pax_global_header00006660000000000000000000000064142642411770014522gustar00rootroot0000000000000052 comment=ce7e38c927948970c219b3d1a0f40ae3458dede3 b2-sdk-python-1.17.3/000077500000000000000000000000001426424117700142145ustar00rootroot00000000000000b2-sdk-python-1.17.3/.github/000077500000000000000000000000001426424117700155545ustar00rootroot00000000000000b2-sdk-python-1.17.3/.github/SUPPORT.md000066400000000000000000000012711426424117700172530ustar00rootroot00000000000000Issues with **B2_Command_line_tool** (problems with the tool itself) should be reported at [B2 CLI issue tracker](https://github.com/Backblaze/B2_Command_Line_Tool/issues). Issuew with **B2 cloud service** (not caused by B2 CLI or sdk) should be reported directly to [Backblaze support](https://help.backblaze.com/hc/en-us/requests/new). Issues with the B2 python sdk should be reported in [b2-sdk-python issue tracker](https://github.com/Backblaze/b2-sdk-python/issues). This should be used by authors of tools that interact with the b2 cloud through **b2-sdk-python**, but also if an issue with some other project (that uses **b2-sdk-python**) is clearly caused by a bug in **b2-sdk-python**. b2-sdk-python-1.17.3/.github/dependabot.yml000066400000000000000000000003611426424117700204040ustar00rootroot00000000000000# Documentation available at # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: - package-ecosystem: "pip" directory: "/" schedule: interval: "daily" b2-sdk-python-1.17.3/.github/no-response.yml000066400000000000000000000013211426424117700205440ustar00rootroot00000000000000# Configuration for probot-no-response - https://github.com/probot/no-response # Number of days of inactivity before an Issue is closed for lack of response daysUntilClose: 14 # Label requiring a response responseRequiredLabel: more-information-needed # Comment to post when closing an Issue for lack of response. Set to `false` to disable closeComment: > This issue has been automatically closed because there has been no response to our request for more information from the original author. With only the information that is currently in the issue, we don't have enough information to take action. Please reach out if you have or find the answers we need so that we can investigate or assist you further. b2-sdk-python-1.17.3/.github/workflows/000077500000000000000000000000001426424117700176115ustar00rootroot00000000000000b2-sdk-python-1.17.3/.github/workflows/cd.yml000066400000000000000000000035701426424117700207270ustar00rootroot00000000000000name: Continuous Delivery on: push: tags: 'v*' # push events to matching v*, i.e. v1.0, v20.15.10 env: PYTHON_DEFAULT_VERSION: "3.10" jobs: deploy: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} B2_PYPI_PASSWORD: ${{ secrets.B2_PYPI_PASSWORD }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v2 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - name: Display Python version run: python -c "import sys; print(sys.version)" - name: Install dependencies run: python -m pip install --upgrade nox pip setuptools - name: Build the distribution id: build run: nox -vs build - name: Read the Changelog id: read-changelog uses: mindsers/changelog-reader-action@v1 with: version: ${{ steps.build.outputs.version }} - name: Create GitHub release id: create-release uses: actions/create-release@v1 with: tag_name: ${{ github.ref }} release_name: ${{ steps.build.outputs.version }} body: ${{ steps.read-changelog.outputs.log_entry }} draft: false prerelease: false - name: Upload the distribution to GitHub uses: actions/upload-release-asset@v1 with: upload_url: ${{ steps.create-release.outputs.upload_url }} asset_path: ${{ steps.build.outputs.asset_path }} asset_name: ${{ steps.build.outputs.asset_name }} asset_content_type: application/gzip - name: Upload the distribution to PyPI if: ${{ env.B2_PYPI_PASSWORD != '' }} uses: pypa/gh-action-pypi-publish@v1.3.1 with: user: __token__ password: ${{ env.B2_PYPI_PASSWORD }} b2-sdk-python-1.17.3/.github/workflows/ci.yml000066400000000000000000000113161426424117700207310ustar00rootroot00000000000000name: Continuous Integration on: push: branches: [master] pull_request: branches: [master] env: PYTHON_DEFAULT_VERSION: "3.10" SKIP_COVERAGE_PYTHON_VERSION_PREFIX: "pypy" jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - uses: codespell-project/actions-codespell@2391250ab05295bddd51e36a8c6295edb6343b0e with: ignore_words_list: datas - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v3 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" - name: Install dependencies run: python -m pip install --upgrade nox pip setuptools - name: Run linters run: nox -vs lint #- name: Validate changelog #- if: ${{ ! startsWith(github.ref, 'refs/heads/dependabot/') }} #- uses: zattoo/changelog@v1 #- with: #- token: ${{ github.token }} build: needs: lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v3 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" - name: Install dependencies run: python -m pip install --upgrade nox pip setuptools - name: Build the distribution run: nox -vs build cleanup_buckets: needs: lint env: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead uses: actions/setup-python@v3 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" - name: Install dependencies if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead run: python -m pip install --upgrade nox pip setuptools - name: Find and remove old buckets if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead run: nox -vs cleanup_old_buckets test: needs: cleanup_buckets env: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11.0-beta.1", "pypy-3.7", "pypy-3.8"] exclude: - os: "macos-latest" python-version: "pypy-3.7" - os: "ubuntu-latest" python-version: "pypy-3.7" - os: "macos-latest" python-version: "pypy-3.8" - os: "windows-latest" python-version: "pypy-3.8" steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} cache: "pip" - name: Install dependencies run: python -m pip install --upgrade nox pip setuptools - name: Run unit tests run: nox -vs unit env: SKIP_COVERAGE: ${{ startsWith(matrix.python-version, env.SKIP_COVERAGE_PYTHON_VERSION_PREFIX) }} - name: Run integration tests if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} run: nox -vs integration -- --dont-cleanup-old-buckets doc: needs: build runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v3 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" - name: Install dependencies env: DEBIAN_FRONTEND: noninteractive run: | sudo apt-get update -y sudo apt-get install -y graphviz plantuml python -m pip install --upgrade nox pip setuptools - name: Build the docs run: nox --non-interactive -vs doc b2-sdk-python-1.17.3/.gitignore000066400000000000000000000001771426424117700162110ustar00rootroot00000000000000*.pyc .codacy-coverage/ .coverage .eggs/ .idea .nox/ .python-version b2sdk.egg-info build coverage.xml dist venv .venv .vscode b2-sdk-python-1.17.3/.readthedocs.yml000066400000000000000000000011201426424117700172740ustar00rootroot00000000000000# .readthedocs.yml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 # Build documentation in the docs/ directory with Sphinx sphinx: configuration: doc/source/conf.py # Optionally build your docs in additional formats such as PDF and ePub formats: all # Optionally set the version of Python and requirements required to build your docs python: version: 3.8 install: - requirements: requirements.txt - method: pip path: . extra_requirements: - doc system_packages: false b2-sdk-python-1.17.3/CHANGELOG.md000066400000000000000000000353661426424117700160420ustar00rootroot00000000000000# Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ## [1.17.3] - 2022-07-15 ### Fixed * Fix `FileVersion._get_upload_headers` when encryption key is `None` ### Infrastructure * Fix download integration tests on non-production environments * Add `B2_DEBUG_HTTP` env variable to enable network-level test debugging * Disable changelog validation temporarily ## [1.17.2] - 2022-06-24 ### Fixed * Fix a race in progress reporter * Fix import of replication ## [1.17.1] - 2022-06-23 [YANKED] ### Fixed * Fix importing scan module ## [1.17.0] - 2022-06-23 [YANKED] As in version 1.16.0, the replication API may still be unstable, however no backward-incompatible changes are planned at this point. ### Added * Add `included_sources` module for keeping track of included modified third-party libraries * Add `include_existing_files` parameter to `ReplicationSetupHelper` ### Changed * Downloading compressed files with `Content-Encoding` header set no longer causes them to be decompressed on the fly - it's an option * Change the per part retry limit from 5 to 20 for data transfer operations. Please note that the retry system is not considered to be a part of the public interface and is subject to be adjusted * Do not wait more than 64 seconds between retry attempts (unless server asks for it) * On longer failures wait an additional (random, up to 1s) amount of time to prevent client synchronization * Flatten `ReplicationConfiguration` interface * Reorder actions of `ReplicationSetupHelper` to avoid zombie rules ### Fixed * Fix: downloading compressed files and decompressing them on the fly now does not cause a TruncatedOutput error * Fix `AccountInfo.is_master_key()` * Fix docstring of `SqliteAccountInfo` * Fix lifecycle rule type in the docs ### Infrastructure * Add 3.11.0-beta.1 to CI * Change Sphinx major version from 5 to 6 * Extract folder/bucket scanning into a new `scan` module * Enable pip cache in CI ## [1.16.0] - 2022-04-27 This release contains a preview of replication support. It allows for basic usage of B2 replication feature (currently in closed beta). As the interface of the sdk (and the server api) may change, the replication support shall be considered PRIVATE interface and should be used with caution. Please consult the documentation on how to safely use the private api interface. Expect substantial amount of work on sdk interface: * The interface of `ReplicationConfiguration` WILL change * The interface of `FileVersion.replication_status` MIGHT change * The interface of `FileVersionDownload` MIGHT change ### Added * Add basic replication support to `Bucket` and `FileVersion` * Add `is_master_key()` method to `AbstractAccountInfo` * Add `readBucketReplications` and `writeBucketReplications` to `ALL_CAPABILITIES` * Add log tracing of `interpret_b2_error` * Add `ReplicationSetupHelper` ### Fixed * Fix license test on Windows * Fix cryptic errors when running integration tests with a non-full key ## [1.15.0] - 2022-04-12 ### Changed * Don't run coverage in pypy in CI * Introduce a common thread worker pool for all downloads * Increase http timeout to 20min (for copy using 5GB parts) * Remove inheritance from object (leftover from python2) * Run unit tests on all CPUs ### Added * Add pypy-3.8 to test matrix * Add support for unverified checksum upload mode * Add dedicated exception for unverified email * Add a parameter to customize `sync_policy_manager` * Add parameters to set the min/max part size for large file upload/copy methods * Add CopySourceTooBig exception * Add an option to set a custom file version class to `FileVersionFactory` * Add an option for B2Api to turn off hash checking for downloaded files * Add an option for B2Api to set write buffer size for `DownloadedFile.save_to` method * Add support for multiple profile files for SqliteAccountInfo ### Fixed * Fix copying objects larger than 1TB * Fix uploading objects larger than 1TB * Fix downloading files with unverified checksum * Fix decoding in filename and file info of `DownloadVersion` * Fix an off-by-one bug and other bugs in the Simulator copy functionality ### Removed * Drop support for Python 3.5 and Python 3.6 ## [1.14.1] - 2022-02-23 ### Security * Fix setting permissions for local sqlite database (thanks to Jan Schejbal for responsible disclosure!) ## [1.14.0] - 2021-12-23 ### Fixed * Relax constraint on arrow to allow for versions >= 1.0.2 ## [1.13.0] - 2021-10-24 ### Added * Add support for Python 3.10 ### Changed * Update a list with all capabilities ### Fixed * Fix pypy selector in CI ## [1.12.0] - 2021-08-06 ### Changed * The `importlib-metadata` requirement is less strictly bound now (just >=3.3.0 for python > 3.5). * `B2Api` `update_file_legal_hold` and `update_file_retention_setting` now return the set values ### Added * `BucketIdNotFound` thrown based on B2 cloud response * `_clone` method to `FileVersion` and `DownloadVersion` * `delete`, `update_legal_hold`, `update_retention` and `download` methods added to `FileVersion` ### Fixed * FileSimulator returns special file info headers properly ### Removed * One unused import. ## [1.11.0] - 2021-06-24 ### Changed * apiver `v2` interface released. `from b2sdk.v2 import ...` is now the recommended import, but `from b2sdk.v1 import ...` works as before ## [1.10.0] - 2021-06-23 ### Added * `get_fresh_state` method added to `FileVersion` and `Bucket` ### Changed * `download_file_*` methods refactored to allow for inspecting DownloadVersion before downloading the whole file * `B2Api.get_file_info` returns a `FileVersion` object in v2 * `B2RawApi` renamed to `B2RawHTTPApi` * `B2HTTP` tests are now common * `B2HttpApiConfig` class introduced to provide parameters like `user_agent_append` to `B2Api` without using internal classes in v2 * `Bucket.update` returns a `Bucket` object in v2 * `Bucket.ls` argument `show_versions` renamed to `latest_only` in v2 * `B2Api` application key methods refactored to operate with dataclasses instead of dicts in v2 * `B2Api.list_keys` is a generator lazily fetching all keys in v2 * `account_id` and `bucket_id` added to FileVersion ### Fixed * Fix EncryptionSetting.from_response_headers * Fix FileVersion.size and FileVersion.mod_time_millis type ambiguity * Old buckets (from past tests) are cleaned up before running integration tests in a single thread ### Removed * Remove deprecated `SyncReport` methods ## [1.9.0] - 2021-06-07 ### Added * `ScanPoliciesManager` is able to filter b2 files by upload timestamp ### Changed * `Synchronizer.make_file_sync_actions` and `Synchronizer.make_folder_sync_actions` were made private in v2 interface * Refactored `sync.file.*File` and `sync.file.*FileVersion` to `sync.path.*SyncPath` in v2 * Refactored `FileVersionInfo` to `FileVersion` in v2 * `ScanPoliciesManager` exclusion interface changed in v2 * `B2Api` unittests for v0, v1 and v2 are now common * `B2Api.cancel_large_file` returns a `FileIdAndName` object instead of a `FileVersion` object in v2 * `FileVersion` has a mandatory `api` parameter in v2 * `B2Folder` holds a handle to B2Api * `Bucket` unit tests for v1 and v2 are now common ### Fixed * Fix call to incorrect internal api in `B2Api.get_download_url_for_file_name` ## [1.8.0] - 2021-05-21 ### Added * Add `get_bucket_name_or_none_from_bucket_id` to `AccountInfo` and `Cache` * Add possibility to change realm during integration tests * Add support for "file locks": file retention, legal hold and default bucket retention ### Fixed * Cleanup sync errors related to directories * Use proper error handling in `ScanPoliciesManager` * Application key restriction message reverted to previous form * Added missing apiver wrappers for FileVersionInfo * Fix crash when Content-Range header is missing * Pin dependency versions appropriately ### Changed * `b2sdk.v1.sync` refactored to reflect `b2sdk.sync` structure * Make `B2Api.get_bucket_by_id` return populated bucket objects in v2 * Add proper support of `recommended_part_size` and `absolute_minimum_part_size` in `AccountInfo` * Refactored `minimum_part_size` to `recommended_part_size` (the value used stays the same) * Encryption settings, types and providers are now part of the public API ### Removed * Remove `Bucket.copy_file` and `Bucket.start_large_file` * Remove `FileVersionInfo.format_ls_entry` and `FileVersionInfo.format_folder_ls_entry` ## [1.7.0] - 2021-04-22 ### Added * Add `__slots__` and `__eq__` to `FileVersionInfo` for memory usage optimization and ease of testing * Add support for SSE-C server-side encryption mode * Add support for `XDG_CONFIG_HOME` for determining the location of `SqliteAccountInfo` db file ### Changed * `BasicSyncEncryptionSettingsProvider` supports different settings sets for reading and writing * Refactored AccountInfo tests to a single file using pytest ### Fixed * Fix clearing cache during `authorize_account` * Fix `ChainedStream` (needed in `Bucket.create_file` etc.) * Make tqdm-based progress reporters less jumpy and easier to read * Fix emerger examples in docs ## [1.6.0] - 2021-04-08 ### Added * Fetch S3-compatible API URL from `authorize_account` ### Fixed * Exclude packages inside the test package when installing * Fix for server response change regarding SSE ## [1.5.0] - 2021-03-25 ### Added * Add `dependabot.yml` * Add support for SSE-B2 server-side encryption mode ### Changed * Add upper version limit for the requirements ### Fixed * Pin `setuptools-scm<6.0` as `>=6.0` doesn't support Python 3.5 ## [1.4.0] - 2021-03-03 ### Changed * Add an ability to provide `bucket_id` filter parameter for `list_buckets` * Add `is_same_key` method to `AccountInfo` * Add upper version limit for arrow dependency, because of a breaking change ### Fixed * Fix docs autogen ## [1.3.0] - 2021-01-13 ### Added * Add custom exception for `403 transaction_cap_exceeded` * Add `get_file_info_by_id` and `get_file_info_by_name` to `Bucket` * `FileNotPresent` and `NonExistentBucket` now subclass new exceptions `FileOrBucketNotFound` and `ResourceNotFound` ### Changed * Fix missing import in the synchronization example * Use `setuptools-scm` for versioning * Clean up CI steps ## [1.2.0] - 2020-11-03 ### Added * Add support for Python 3.9 * Support for bucket to bucket sync * Add a possibility to append a string to the User-Agent in `B2Http` ### Changed * Change default fetch count for `ls` to 10000 ### Removed * Drop Python 2 and Python 3.4 support :tada: * Remove `--prefix` from `ls` (it didn't really work, use `folderName` argument) ### Fixed * Allow to set an empty bucket info during the update * Fix docs generation in CI ## [1.1.4] - 2020-07-15 ### Added * Allow specifying custom realm in B2Session.authorize_account ## [1.1.2] - 2020-07-06 ### Fixed * Fix upload part for file range on Python 2.7 ## [1.1.0] - 2020-06-24 ### Added * Add `list_file_versions` method to buckets. * Add server-side copy support for large files * Add ability to synthesize objects from local and remote sources * Add AuthInfoCache, InMemoryCache and AbstractCache to public interface * Add ability to filter in ScanPoliciesManager based on modification time * Add ScanPoliciesManager and SyncReport to public interface * Add md5 checksum to FileVersionInfo * Add more keys to dicts returned by as_dict() methods ### Changed * Make sync treat hidden files as deleted * Ignore urllib3 "connection pool is full" warning ### Removed * Remove arrow warnings caused by https://github.com/crsmithdev/arrow/issues/612 ### Fixed * Fix handling of modification time of files ## [1.0.2] - 2019-10-15 ### Changed * Remove upper version limit for arrow dependency ## [1.0.0] - 2019-10-03 ### Fixed * Minor bug fix. ## [1.0.0-rc1] - 2019-07-09 ### Deprecated * Deprecate some transitional method names to v0 in preparation for v1.0.0. ## [0.1.10] - 2019-07-09 ### Removed * Remove a parameter (which did nothing, really) from `b2sdk.v1.Bucket.copy_file` signature ## [0.1.8] - 2019-06-28 ### Added * Add support for b2_copy_file * Add support for `prefix` parameter on ls-like calls ## [0.1.6] - 2019-04-24 ### Changed * Rename account ID for authentication to application key ID. Account ID is still backwards compatible, only the terminology has changed. ### Fixed * Fix transferer crashing on empty file download attempt ## [0.1.4] - 2019-04-04 ### Added Initial official release of SDK as a separate package (until now it was a part of B2 CLI) [Unreleased]: https://github.com/Backblaze/b2-sdk-python/compare/v1.17.3...HEAD [1.17.3]: https://github.com/Backblaze/b2-sdk-python/compare/v1.17.2...v1.17.3 [1.17.2]: https://github.com/Backblaze/b2-sdk-python/compare/v1.17.1...v1.17.2 [1.17.1]: https://github.com/Backblaze/b2-sdk-python/compare/v1.17.0...v1.17.1 [1.17.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.16.0...v1.17.0 [1.16.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.15.0...v1.16.0 [1.15.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.14.1...v1.15.0 [1.14.1]: https://github.com/Backblaze/b2-sdk-python/compare/v1.14.0...v1.14.1 [1.14.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.13.0...v1.14.0 [1.13.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.12.0...v1.13.0 [1.12.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.11.0...v1.12.0 [1.11.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.10.0...v1.11.0 [1.10.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.9.0...v1.10.0 [1.9.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.8.0...v1.9.0 [1.8.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.7.0...v1.8.0 [1.7.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.6.0...v1.7.0 [1.6.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.5.0...v1.6.0 [1.5.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.4.0...v1.5.0 [1.4.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.3.0...v1.4.0 [1.3.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.2.0...v1.3.0 [1.2.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.1.4...v1.2.0 [1.1.4]: https://github.com/Backblaze/b2-sdk-python/compare/v1.1.2...v1.1.4 [1.1.2]: https://github.com/Backblaze/b2-sdk-python/compare/v1.1.0...v1.1.2 [1.1.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.0.2...v1.1.0 [1.0.2]: https://github.com/Backblaze/b2-sdk-python/compare/v1.0.0...v1.0.2 [1.0.0]: https://github.com/Backblaze/b2-sdk-python/compare/v1.0.0-rc1...v1.0.0 [1.0.0-rc1]: https://github.com/Backblaze/b2-sdk-python/compare/v0.1.10...v1.0.0-rc1 [0.1.10]: https://github.com/Backblaze/b2-sdk-python/compare/v0.1.8...v0.1.10 [0.1.8]: https://github.com/Backblaze/b2-sdk-python/compare/v0.1.6...v0.1.8 [0.1.6]: https://github.com/Backblaze/b2-sdk-python/compare/v0.1.4...v0.1.6 [0.1.4]: https://github.com/Backblaze/b2-sdk-python/compare/4fd290c...v0.1.4 b2-sdk-python-1.17.3/CONTRIBUTING.md000066400000000000000000000062421426424117700164510ustar00rootroot00000000000000# Contributing to B2 Python SDK We encourage outside contributors to perform changes on our codebase. Many such changes have been merged already. In order to make it easier to contribute, core developers of this project: * provide guidance (through the issue reporting system) * provide tool assisted code review (through the Pull Request system) * maintain a set of unit tests * maintain a set of integration tests (run with a production cloud) * maintain development automation tools using [nox](https://github.com/theacodes/nox) that can easily: * format the code using [yapf](https://github.com/google/yapf) * runs linters to find subtle/potential issues with maintainability * run the test suite on multiple Python versions using [pytest](https://github.com/pytest-dev/pytest) * maintain Continuous Integration (by using GitHub Actions) that: * runs all sorts of linters * checks if the Python distribution can be built * runs all tests on a matrix of 6 versions of Python (including pypy) and 3 operating systems (Linux, Mac OS X and Windows) * checks if the documentation can be built properly * maintain other Continuous Integration tools (coverage tracker) ## Developer Info You'll need to have [nox](https://github.com/theacodes/nox) installed: * `pip install nox` With `nox`, you can run different sessions (default are `lint` and `test`): * `format` -> Format the code. * `lint` -> Run linters. * `test` (`test-3.7`, `test-3.8`, `test-3.9`, `test-3.10`) -> Run test suite. * `cover` -> Perform coverage analysis. * `build` -> Build the distribution. * `deploy` -> Deploy the distribution to the PyPi. * `doc` -> Build the documentation. * `doc_cover` -> Perform coverage analysis for the documentation. For example: $ nox -s format nox > Running session format nox > Creating virtual environment (virtualenv) using python3.10 in .nox/format ... $ nox -s format nox > Running session format nox > Re-using existing virtual environment at .nox/format. ... $ nox --no-venv -s format nox > Running session format ... Sessions `test` ,`unit`, and `integration` can run on many Python versions, 3.7-3.10 by default. Sessions other than `test` use the last given Python version, 3.10 by default. You can change it: export NOX_PYTHONS=3.7,3.8 With the above setting, session `test` will run on Python 3.7 and 3.8, and all other sessions on Python 3.8. Given Python interpreters should be installed in the operating system or via [pyenv](https://github.com/pyenv/pyenv). ## Linting To run all available linters: nox -s lint ## Testing To run all tests on every available Python version: nox -s test To run all tests on a specific version: nox -s test-3.10 To run just unit tests: nox -s unit-3.10 To run just integration tests: export B2_TEST_APPLICATION_KEY=your_app_key export B2_TEST_APPLICATION_KEY_ID=your_app_key_id nox -s integration-3.10 To run tests by keyword expressions: nox -s unit-3.10 -- -k keyword ## Documentation To build the documentation and watch for changes (including the source code): nox -s doc To just build the documentation: nox --non-interactive -s doc b2-sdk-python-1.17.3/LICENSE000066400000000000000000000272731426424117700152340ustar00rootroot00000000000000Backblaze wants developers and organization to copy and re-use our code examples, so we make the samples available by several different licenses. One option is the MIT license (below). Other options are available here: https://www.backblaze.com/using_b2_code.html The MIT License (MIT) Copyright (c) 2015 Backblaze Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Backblaze has also utilized other open source code (psf/requests) licensed under the Apache License, Version 2.0, January 2004. That Apache License requires that we also provide you will a copy of the Apache License (below, and accessible at https://www.apache.org/licenses/LICENSE-2.0). Note that Backblaze does not license its own code to you under the Apache License, but simply provides you a copy of such Apache License to comply with its own license obligations. Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. b2-sdk-python-1.17.3/MANIFEST.in000066400000000000000000000000511426424117700157460ustar00rootroot00000000000000include requirements.txt include LICENSE b2-sdk-python-1.17.3/README.md000066400000000000000000000042631426424117700155000ustar00rootroot00000000000000# B2 Python SDK  [![Continuous Integration](https://github.com/Backblaze/b2-sdk-python/workflows/Continuous%20Integration/badge.svg)](https://github.com/Backblaze/b2-sdk-python/actions?query=workflow%3A%22Continuous+Integration%22) [![License](https://img.shields.io/pypi/l/b2sdk.svg?label=License)](https://pypi.python.org/pypi/b2) [![python versions](https://img.shields.io/pypi/pyversions/b2sdk.svg?label=python%20versions)](https://pypi.python.org/pypi/b2sdk) [![PyPI version](https://img.shields.io/pypi/v/b2sdk.svg?label=PyPI%20version)](https://pypi.python.org/pypi/b2sdk) [![Docs](https://readthedocs.org/projects/b2-sdk-python/badge/?version=master)](https://b2-sdk-python.readthedocs.io/en/master/) This repository contains a client library and a few handy utilities for easy access to all of the capabilities of B2 Cloud Storage. [B2 command-line tool](https://github.com/Backblaze/B2_Command_Line_Tool) is an example of how it can be used to provide command-line access to the B2 service, but there are many possible applications (including FUSE filesystems, storage backend drivers for backup applications etc). # Documentation The latest documentation is available on [Read the Docs](https://b2-sdk-python.readthedocs.io). # Installation The sdk can be installed with: pip install b2sdk # Version policy b2sdk follows [Semantic Versioning](https://semver.org/) policy, so in essence the version number is MAJOR.MINOR.PATCH (for example 1.2.3) and: - we increase MAJOR version when we make incompatible API changes - we increase MINOR version when we add functionality in a backwards-compatible manner, and - we increase PATCH version when we make backwards-compatible bug fixes (unless someone relies on the undocumented behavior of a fixed bug) Therefore when setting up b2sdk as a dependency, please make sure to match the version appropriately, for example you could put this in your `requirements.txt` to make sure your code is compatible with the `b2sdk` version your user will get from pypi: ``` b2sdk>=0.0.0,<1.0.0 ``` # Release History Please refer to the [changelog](CHANGELOG.md). # Developer Info Please see our [contributing guidelines](CONTRIBUTING.md). b2-sdk-python-1.17.3/README.release.md000066400000000000000000000023001426424117700171050ustar00rootroot00000000000000# Release Process - Get the Nox: - `pip install -U nox` - Update the release history in `CHANGELOG.md`: - Change "Unreleased" to the current release version and date. - Create empty "Unreleased" section. - Add proper link to the new release (at the bottom of the file). Use GitHub [compare feature](https://docs.github.com/en/free-pro-team@latest/github/committing-changes-to-your-project/comparing-commits#comparing-tags) between two tags. - Update "Unreleased" link (at the bottom of the file). - Run linters and tests: - `export B2_TEST_APPLICATION_KEY=your_app_key` - `export B2_TEST_APPLICATION_KEY_ID=your_app_key_id` - `nox -x` - Build docs locally: - `nox --non-interactive -xs doc` - Commit and push to GitHub, then wait for CI workflow to complete successfully. - No need to make a branch. Push straight to `master`. - Tag in git and push tag to origin. (Version tags look like "v0.4.6".) - `git tag vx.x.x` - `git push origin vx.x.x` - Wait for CD workflow to complete successfully. - Verify that the GitHub release is created - Verify that the release has been uploaded to the PyPI - Install using pip and verify that it gets the correct version: - `pip install -U b2sdk` b2-sdk-python-1.17.3/b2sdk/000077500000000000000000000000001426424117700152215ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/LICENSE000077700000000000000000000000001426424117700174412../LICENSEustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/__init__.py000066400000000000000000000017721426424117700173410ustar00rootroot00000000000000###################################################################### # # File: b2sdk/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # Set default logging handler to avoid "No handler found" warnings. import logging logging.getLogger(__name__).addHandler(logging.NullHandler()) class UrllibWarningFilter: def filter(self, record): return record.msg != "Connection pool is full, discarding connection: %s" logging.getLogger('urllib3.connectionpool').addFilter(UrllibWarningFilter()) import b2sdk.version __version__ = b2sdk.version.VERSION assert __version__ # PEP-0396 # https://github.com/crsmithdev/arrow/issues/612 - To get rid of the ArrowParseWarning messages in 0.14.3 onward. try: from arrow.factory import ArrowParseWarning except ImportError: pass else: import warnings warnings.simplefilter("ignore", ArrowParseWarning) b2-sdk-python-1.17.3/b2sdk/__main__.py000066400000000000000000000004341426424117700173140ustar00rootroot00000000000000###################################################################### # # File: b2sdk/__main__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/_pyinstaller/000077500000000000000000000000001426424117700177265ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/_pyinstaller/__init__.py000066400000000000000000000010041426424117700220320ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_pyinstaller/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os def get_hook_dirs(): """Get hooks directories for pyinstaller. More info about the hooks: https://pyinstaller.readthedocs.io/en/stable/hooks.html """ return [os.path.dirname(__file__)] b2-sdk-python-1.17.3/b2sdk/_pyinstaller/hook-b2sdk.py000066400000000000000000000005761426424117700222530ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_pyinstaller/hook-b2sdk.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from PyInstaller.utils.hooks import copy_metadata datas = copy_metadata('b2sdk') b2-sdk-python-1.17.3/b2sdk/_v3/000077500000000000000000000000001426424117700157105ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/_v3/__init__.py000066400000000000000000000225251426424117700200270ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_v3/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # this file maps the external interface into internal interface # it will come handy if we ever need to move something # core from b2sdk.api import B2Api from b2sdk.api import Services from b2sdk.bucket import Bucket from b2sdk.bucket import BucketFactory from b2sdk.raw_api import ALL_CAPABILITIES, REALM_URLS # encryption from b2sdk.encryption.setting import EncryptionSetting from b2sdk.encryption.setting import EncryptionSettingFactory from b2sdk.encryption.setting import EncryptionKey from b2sdk.encryption.setting import SSE_NONE, SSE_B2_AES, UNKNOWN_KEY_ID from b2sdk.encryption.types import EncryptionAlgorithm from b2sdk.encryption.types import EncryptionMode from b2sdk.http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME # account info from b2sdk.account_info.abstract import AbstractAccountInfo from b2sdk.account_info.in_memory import InMemoryAccountInfo from b2sdk.account_info.sqlite_account_info import SqliteAccountInfo from b2sdk.account_info.sqlite_account_info import B2_ACCOUNT_INFO_ENV_VAR from b2sdk.account_info.sqlite_account_info import B2_ACCOUNT_INFO_DEFAULT_FILE from b2sdk.account_info.sqlite_account_info import B2_ACCOUNT_INFO_PROFILE_FILE from b2sdk.account_info.sqlite_account_info import XDG_CONFIG_HOME_ENV_VAR from b2sdk.account_info.stub import StubAccountInfo from b2sdk.account_info.upload_url_pool import UploadUrlPool from b2sdk.account_info.upload_url_pool import UrlPoolAccountInfo # version & version utils from b2sdk.version import VERSION, USER_AGENT from b2sdk.version_utils import rename_argument, rename_function # utils from b2sdk.utils import ( b2_url_encode, b2_url_decode, choose_part_ranges, current_time_millis, fix_windows_path_limit, format_and_scale_fraction, format_and_scale_number, hex_sha1_of_stream, hex_sha1_of_bytes, hex_sha1_of_file, TempDir, ) from b2sdk.utils import trace_call # data classes from b2sdk.application_key import ApplicationKey from b2sdk.application_key import BaseApplicationKey from b2sdk.application_key import FullApplicationKey from b2sdk.file_version import DownloadVersion from b2sdk.file_version import DownloadVersionFactory from b2sdk.file_version import FileIdAndName from b2sdk.file_version import FileVersion from b2sdk.file_version import FileVersionFactory from b2sdk.large_file.part import Part from b2sdk.large_file.unfinished_large_file import UnfinishedLargeFile from b2sdk.utils.range_ import Range # file lock from b2sdk.file_lock import BucketRetentionSetting from b2sdk.file_lock import FileLockConfiguration from b2sdk.file_lock import FileRetentionSetting from b2sdk.file_lock import LegalHold from b2sdk.file_lock import NO_RETENTION_BUCKET_SETTING from b2sdk.file_lock import NO_RETENTION_FILE_SETTING from b2sdk.file_lock import RetentionMode from b2sdk.file_lock import RetentionPeriod from b2sdk.file_lock import UNKNOWN_BUCKET_RETENTION from b2sdk.file_lock import UNKNOWN_FILE_LOCK_CONFIGURATION from b2sdk.file_lock import UNKNOWN_FILE_RETENTION_SETTING # progress reporting from b2sdk.progress import AbstractProgressListener from b2sdk.progress import DoNothingProgressListener from b2sdk.progress import ProgressListenerForTest from b2sdk.progress import SimpleProgressListener from b2sdk.progress import TqdmProgressListener from b2sdk.progress import make_progress_listener # raw_simulator from b2sdk.raw_simulator import BucketSimulator from b2sdk.raw_simulator import FakeResponse from b2sdk.raw_simulator import FileSimulator from b2sdk.raw_simulator import KeySimulator from b2sdk.raw_simulator import PartSimulator from b2sdk.raw_simulator import RawSimulator # raw_api from b2sdk.raw_api import AbstractRawApi from b2sdk.raw_api import B2RawHTTPApi from b2sdk.raw_api import MetadataDirectiveMode # stream from b2sdk.stream.chained import StreamOpener from b2sdk.stream.progress import AbstractStreamWithProgress from b2sdk.stream import RangeOfInputStream from b2sdk.stream import ReadingStreamWithProgress from b2sdk.stream import StreamWithHash from b2sdk.stream import WritingStreamWithProgress # source / destination from b2sdk.transfer.inbound.downloaded_file import DownloadedFile from b2sdk.transfer.inbound.downloaded_file import MtimeUpdatedFile from b2sdk.transfer.inbound.download_manager import DownloadManager from b2sdk.transfer.outbound.outbound_source import OutboundTransferSource from b2sdk.transfer.outbound.copy_source import CopySource from b2sdk.transfer.outbound.upload_source import AbstractUploadSource from b2sdk.transfer.outbound.upload_source import UploadSourceBytes from b2sdk.transfer.outbound.upload_source import UploadSourceLocalFile from b2sdk.transfer.outbound.upload_source import UploadSourceLocalFileRange from b2sdk.transfer.outbound.upload_source import UploadSourceStream from b2sdk.transfer.outbound.upload_source import UploadSourceStreamRange from b2sdk.transfer.outbound.upload_manager import UploadManager from b2sdk.transfer.emerge.planner.upload_subpart import CachedBytesStreamOpener from b2sdk.transfer.emerge.write_intent import WriteIntent # transfer from b2sdk.transfer.inbound.downloader.abstract import AbstractDownloader from b2sdk.transfer.outbound.large_file_upload_state import LargeFileUploadState from b2sdk.transfer.inbound.downloader.parallel import ParallelDownloader from b2sdk.transfer.inbound.downloader.parallel import PartToDownload from b2sdk.transfer.inbound.downloader.parallel import WriterThread from b2sdk.transfer.outbound.progress_reporter import PartProgressReporter from b2sdk.transfer.inbound.downloader.simple import SimpleDownloader # sync from b2sdk.sync.action import AbstractAction from b2sdk.sync.action import B2CopyAction from b2sdk.sync.action import B2DeleteAction from b2sdk.sync.action import B2DownloadAction from b2sdk.sync.action import B2HideAction from b2sdk.sync.action import B2UploadAction from b2sdk.sync.action import LocalDeleteAction from b2sdk.sync.exception import IncompleteSync from b2sdk.sync.policy import AbstractFileSyncPolicy from b2sdk.sync.policy import CompareVersionMode from b2sdk.sync.policy import NewerFileSyncMode from b2sdk.sync.policy import DownAndDeletePolicy from b2sdk.sync.policy import DownAndKeepDaysPolicy from b2sdk.sync.policy import DownPolicy from b2sdk.sync.policy import CopyPolicy from b2sdk.sync.policy import CopyAndDeletePolicy from b2sdk.sync.policy import CopyAndKeepDaysPolicy from b2sdk.sync.policy import UpAndDeletePolicy from b2sdk.sync.policy import UpAndKeepDaysPolicy from b2sdk.sync.policy import UpPolicy from b2sdk.sync.policy import make_b2_keep_days_actions from b2sdk.sync.policy_manager import SyncPolicyManager from b2sdk.sync.policy_manager import POLICY_MANAGER from b2sdk.sync.report import SyncFileReporter from b2sdk.sync.report import SyncReport from b2sdk.sync.sync import KeepOrDeleteMode from b2sdk.sync.sync import Synchronizer from b2sdk.sync.encryption_provider import AbstractSyncEncryptionSettingsProvider from b2sdk.sync.encryption_provider import BasicSyncEncryptionSettingsProvider from b2sdk.sync.encryption_provider import ServerDefaultSyncEncryptionSettingsProvider from b2sdk.sync.encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER # scan from b2sdk.scan.exception import EnvironmentEncodingError from b2sdk.scan.exception import InvalidArgument from b2sdk.scan.folder import AbstractFolder from b2sdk.scan.folder import B2Folder from b2sdk.scan.folder import LocalFolder from b2sdk.scan.folder_parser import parse_folder from b2sdk.scan.path import AbstractPath, B2Path, LocalPath from b2sdk.scan.policies import convert_dir_regex_to_dir_prefix_regex from b2sdk.scan.policies import DEFAULT_SCAN_MANAGER from b2sdk.scan.policies import IntegerRange from b2sdk.scan.policies import RegexSet from b2sdk.scan.policies import ScanPoliciesManager from b2sdk.scan.report import ProgressReport from b2sdk.scan.scan import zip_folders from b2sdk.scan.scan import AbstractScanResult from b2sdk.scan.scan import AbstractScanReport from b2sdk.scan.scan import CountAndSampleScanReport # replication from b2sdk.replication.setting import ReplicationConfigurationFactory from b2sdk.replication.setting import ReplicationConfiguration from b2sdk.replication.setting import ReplicationRule from b2sdk.replication.types import ReplicationStatus from b2sdk.replication.setup import ReplicationSetupHelper from b2sdk.replication.monitoring import ReplicationScanResult from b2sdk.replication.monitoring import ReplicationReport from b2sdk.replication.monitoring import ReplicationMonitor # other from b2sdk.included_sources import get_included_sources from b2sdk.b2http import B2Http from b2sdk.api_config import B2HttpApiConfig from b2sdk.api_config import DEFAULT_HTTP_API_CONFIG from b2sdk.b2http import ClockSkewHook from b2sdk.b2http import HttpCallback from b2sdk.b2http import ResponseContextManager from b2sdk.bounded_queue_executor import BoundedQueueExecutor from b2sdk.cache import AbstractCache from b2sdk.cache import AuthInfoCache from b2sdk.cache import DummyCache from b2sdk.cache import InMemoryCache from b2sdk.http_constants import SRC_LAST_MODIFIED_MILLIS from b2sdk.session import B2Session from b2sdk.utils.thread_pool import ThreadPoolMixin b2-sdk-python-1.17.3/b2sdk/_v3/exception.py000066400000000000000000000124371426424117700202670ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_v3/exception.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.account_info.exception import AccountInfoError from b2sdk.account_info.exception import CorruptAccountInfo from b2sdk.account_info.exception import MissingAccountData from b2sdk.exception import AccessDenied from b2sdk.exception import AlreadyFailed from b2sdk.exception import B2ConnectionError from b2sdk.exception import B2Error from b2sdk.exception import B2HttpCallbackException from b2sdk.exception import B2HttpCallbackPostRequestException from b2sdk.exception import B2HttpCallbackPreRequestException from b2sdk.exception import B2RequestTimeout from b2sdk.exception import B2RequestTimeoutDuringUpload from b2sdk.exception import B2SimpleError from b2sdk.exception import BadDateFormat from b2sdk.exception import BadFileInfo from b2sdk.exception import BadJson from b2sdk.exception import BadRequest from b2sdk.exception import BadUploadUrl from b2sdk.exception import BucketIdNotFound from b2sdk.exception import BrokenPipe from b2sdk.exception import BucketNotAllowed from b2sdk.exception import CapabilityNotAllowed from b2sdk.exception import CapExceeded from b2sdk.exception import ChecksumMismatch from b2sdk.exception import ClockSkew from b2sdk.exception import Conflict from b2sdk.exception import ConnectionReset from b2sdk.exception import CopyArgumentsMismatch from b2sdk.exception import DestFileNewer from b2sdk.exception import DuplicateBucketName from b2sdk.exception import FileAlreadyHidden from b2sdk.exception import FileNameNotAllowed from b2sdk.exception import FileNotPresent from b2sdk.exception import FileSha1Mismatch from b2sdk.exception import InvalidAuthToken from b2sdk.exception import InvalidMetadataDirective from b2sdk.exception import InvalidRange from b2sdk.exception import InvalidUploadSource from b2sdk.exception import MaxFileSizeExceeded from b2sdk.exception import MaxRetriesExceeded from b2sdk.exception import MissingPart from b2sdk.exception import NonExistentBucket from b2sdk.exception import NotAllowedByAppKeyError from b2sdk.exception import PartSha1Mismatch from b2sdk.exception import RestrictedBucket from b2sdk.exception import RetentionWriteError from b2sdk.exception import ServiceError from b2sdk.exception import StorageCapExceeded from b2sdk.exception import TooManyRequests from b2sdk.exception import TransientErrorMixin from b2sdk.exception import TransactionCapExceeded from b2sdk.exception import TruncatedOutput from b2sdk.exception import Unauthorized from b2sdk.exception import UnexpectedCloudBehaviour from b2sdk.exception import UnknownError from b2sdk.exception import UnknownHost from b2sdk.exception import UnrecognizedBucketType from b2sdk.exception import UnsatisfiableRange from b2sdk.exception import UnusableFileName from b2sdk.exception import SSECKeyIdMismatchInCopy from b2sdk.exception import SSECKeyError from b2sdk.exception import WrongEncryptionModeForBucketDefault from b2sdk.exception import interpret_b2_error from b2sdk.sync.exception import IncompleteSync from b2sdk.scan.exception import UnableToCreateDirectory from b2sdk.scan.exception import EmptyDirectory from b2sdk.scan.exception import EnvironmentEncodingError from b2sdk.scan.exception import InvalidArgument from b2sdk.scan.exception import NotADirectory from b2sdk.scan.exception import UnsupportedFilename from b2sdk.scan.exception import check_invalid_argument __all__ = ( 'AccessDenied', 'AccountInfoError', 'AlreadyFailed', 'B2ConnectionError', 'B2Error', 'B2HttpCallbackException', 'B2HttpCallbackPostRequestException', 'B2HttpCallbackPreRequestException', 'B2RequestTimeout', 'B2RequestTimeoutDuringUpload', 'B2SimpleError', 'BadDateFormat', 'BadFileInfo', 'BadJson', 'BadRequest', 'BadUploadUrl', 'BrokenPipe', 'BucketIdNotFound', 'BucketNotAllowed', 'CapabilityNotAllowed', 'CapExceeded', 'ChecksumMismatch', 'ClockSkew', 'Conflict', 'ConnectionReset', 'CopyArgumentsMismatch', 'CorruptAccountInfo', 'DestFileNewer', 'DuplicateBucketName', 'EmptyDirectory', 'EnvironmentEncodingError', 'FileAlreadyHidden', 'FileNameNotAllowed', 'FileNotPresent', 'FileSha1Mismatch', 'IncompleteSync', 'InvalidArgument', 'InvalidAuthToken', 'InvalidMetadataDirective', 'InvalidRange', 'InvalidUploadSource', 'MaxFileSizeExceeded', 'MaxRetriesExceeded', 'MissingAccountData', 'MissingPart', 'NonExistentBucket', 'NotADirectory', 'NotAllowedByAppKeyError', 'PartSha1Mismatch', 'RestrictedBucket', 'RetentionWriteError', 'ServiceError', 'StorageCapExceeded', 'TooManyRequests', 'TransactionCapExceeded', 'TransientErrorMixin', 'TruncatedOutput', 'Unauthorized', 'UnexpectedCloudBehaviour', 'UnknownError', 'UnknownHost', 'UnrecognizedBucketType', 'UnableToCreateDirectory', 'UnsupportedFilename', 'UnsatisfiableRange', 'UnusableFileName', 'interpret_b2_error', 'check_invalid_argument', 'SSECKeyIdMismatchInCopy', 'SSECKeyError', 'WrongEncryptionModeForBucketDefault', ) b2-sdk-python-1.17.3/b2sdk/account_info/000077500000000000000000000000001426424117700176705ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/account_info/__init__.py000066400000000000000000000005611426424117700220030ustar00rootroot00000000000000###################################################################### # # File: b2sdk/account_info/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from .in_memory import InMemoryAccountInfo assert InMemoryAccountInfo b2-sdk-python-1.17.3/b2sdk/account_info/abstract.py000066400000000000000000000302161426424117700220470ustar00rootroot00000000000000###################################################################### # # File: b2sdk/account_info/abstract.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import abstractmethod from typing import Optional from b2sdk.account_info import exception from b2sdk.raw_api import ALL_CAPABILITIES from b2sdk.utils import B2TraceMetaAbstract, limit_trace_arguments class AbstractAccountInfo(metaclass=B2TraceMetaAbstract): """ Abstract class for a holder for all account-related information that needs to be kept between API calls and between invocations of the program. This includes: account ID, application key ID, application key, auth tokens, API URL, download URL, and uploads URLs. This class must be THREAD SAFE because it may be used by multiple threads running in the same Python process. It also needs to be safe against multiple processes running at the same time. """ # The 'allowed' structure to use for old account info that was saved without 'allowed'. DEFAULT_ALLOWED = dict( bucketId=None, bucketName=None, capabilities=ALL_CAPABILITIES, namePrefix=None, ) @classmethod def all_capabilities(cls): """ Return a list of all possible capabilities. :rtype: list """ return ALL_CAPABILITIES @abstractmethod def clear(self): """ Remove all stored information. """ @abstractmethod @limit_trace_arguments(only=['self']) def refresh_entire_bucket_name_cache(self, name_id_iterable): """ Remove all previous name-to-id mappings and stores new ones. :param iterable name_id_iterable: an iterable of tuples of the form (name, id) """ @abstractmethod def remove_bucket_name(self, bucket_name): """ Remove one entry from the bucket name cache. :param str bucket_name: a bucket name """ @abstractmethod def save_bucket(self, bucket): """ Remember the ID for the given bucket name. :param b2sdk.v2.Bucket bucket: a Bucket object """ @abstractmethod def get_bucket_id_or_none_from_bucket_name(self, bucket_name): """ Look up the bucket ID for the given bucket name. :param str bucket_name: a bucket name :return bucket ID or None: :rtype: str, None """ @abstractmethod def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> Optional[str]: """ Look up the bucket name for the given bucket id. """ @abstractmethod def clear_bucket_upload_data(self, bucket_id): """ Remove all upload URLs for the given bucket. :param str bucket_id: a bucket ID """ def is_same_key(self, application_key_id, realm): """ Check whether cached application key is the same as the one provided. :param str application_key_id: application key ID :param str realm: authorization realm :rtype: bool """ try: return self.get_application_key_id() == application_key_id and self.get_realm() == realm except exception.MissingAccountData: return False def is_same_account(self, account_id: str, realm: str) -> bool: """ Check whether cached account is the same as the one provided. :param str account_id: account ID :param str realm: authorization realm :rtype: bool """ try: return self.get_account_id() == account_id and self.get_realm() == realm except exception.MissingAccountData: return False def is_master_key(self) -> bool: application_key_id = self.get_application_key_id() account_id = self.get_account_id() new_style_master_key_suffix = '0000000000' if account_id == application_key_id: return True # old style if len(application_key_id ) == (3 + len(account_id) + len(new_style_master_key_suffix)): # 3 for cluster id # new style if application_key_id.endswith(account_id + new_style_master_key_suffix): return True return False @abstractmethod def get_account_id(self): """ Return account ID or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_application_key_id(self): """ Return the application key ID used to authenticate. :rtype: str """ @abstractmethod def get_account_auth_token(self): """ Return account_auth_token or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_api_url(self): """ Return api_url or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_application_key(self): """ Return application_key or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_download_url(self): """ Return download_url or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_realm(self): """ Return realm or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_recommended_part_size(self): """ Return the recommended number of bytes in a part of a large file. :return: number of bytes :rtype: int """ @abstractmethod def get_absolute_minimum_part_size(self): """ Return the absolute minimum number of bytes in a part of a large file. :return: number of bytes :rtype: int """ @abstractmethod def get_allowed(self): """ An 'allowed' dict, as returned by ``b2_authorize_account``. Never ``None``; for account info that was saved before 'allowed' existed, returns :attr:`DEFAULT_ALLOWED`. :rtype: dict """ @abstractmethod def get_s3_api_url(self): """ Return s3_api_url or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @limit_trace_arguments( only=[ 'self', 'api_url', 'download_url', 'recommended_part_size', 'absolute_minimum_part_size', 'realm', 's3_api_url', ] ) def set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id, ): """ Check permission correctness and stores the results of ``b2_authorize_account``. The allowed structure is the one returned by ``b2_authorize_account``, e.g. .. code-block:: python { "absoluteMinimumPartSize": 5000000, "accountId": "YOUR_ACCOUNT_ID", "allowed": { "bucketId": "BUCKET_ID", "bucketName": "BUCKET_NAME", "capabilities": [ "listBuckets", "listFiles", "readFiles", "shareFiles", "writeFiles", "deleteFiles" ], "namePrefix": null }, "apiUrl": "https://apiNNN.backblazeb2.com", "authorizationToken": "4_0022623512fc8f80000000001_0186e431_d18d02_acct_tH7VW03boebOXayIc43-sxptpfA=", "downloadUrl": "https://f002.backblazeb2.com", "recommendedPartSize": 100000000, "s3ApiUrl": "https://s3.us-west-NNN.backblazeb2.com" } For keys with bucket restrictions, the name of the bucket is looked up and stored as well. The console_tool does everything by bucket name, so it's convenient to have the restricted bucket name handy. :param str account_id: user account ID :param str auth_token: user authentication token :param str api_url: an API URL :param str download_url: path download URL :param int recommended_part_size: recommended size of a file part :param int absolute_minimum_part_size: minimum size of a file part :param str application_key: application key :param str realm: a realm to authorize account in :param dict allowed: the structure to use for old account info that was saved without 'allowed' :param str application_key_id: application key ID :param str s3_api_url: S3-compatible API URL .. versionchanged:: 0.1.5 `account_id_or_app_key_id` renamed to `application_key_id` """ if allowed is None: allowed = self.DEFAULT_ALLOWED assert self.allowed_is_valid(allowed) self._set_auth_data( account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id ) @classmethod def allowed_is_valid(cls, allowed): """ Make sure that all of the required fields are present, and that bucketId is set if bucketName is. If the bucketId is for a bucket that no longer exists, or the capabilities do not allow for listBuckets, then we will not have a bucketName. :param dict allowed: the structure to use for old account info that was saved without 'allowed' :rtype: bool """ return ( ('bucketId' in allowed) and ('bucketName' in allowed) and ((allowed['bucketId'] is not None) or (allowed['bucketName'] is None)) and ('capabilities' in allowed) and ('namePrefix' in allowed) ) @abstractmethod def _set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id ): """ Actually store the auth data. Can assume that 'allowed' is present and valid. All of the information returned by ``b2_authorize_account`` is saved, because all of it is needed at some point. """ @abstractmethod def take_bucket_upload_url(self, bucket_id): """ Return a pair (upload_url, upload_auth_token) that has been removed from the pool for this bucket, or (None, None) if there are no more left. :param str bucket_id: a bucket ID :rtype: tuple """ @abstractmethod @limit_trace_arguments(only=['self', 'bucket_id']) def put_bucket_upload_url(self, bucket_id, upload_url, upload_auth_token): """ Add an (upload_url, upload_auth_token) pair to the pool available for the bucket. :param str bucket_id: a bucket ID :param str upload_url: an upload URL :param str upload_auth_token: an upload authentication token :rtype: tuple """ @abstractmethod @limit_trace_arguments(only=['self']) def put_large_file_upload_url(self, file_id, upload_url, upload_auth_token): """ Put a large file upload URL into a pool. :param str file_id: a file ID :param str upload_url: an upload URL :param str upload_auth_token: an upload authentication token """ pass @abstractmethod def take_large_file_upload_url(self, file_id): """ Take the chosen large file upload URL from the pool. :param str file_id: a file ID """ pass @abstractmethod def clear_large_file_upload_urls(self, file_id): """ Clear the pool of URLs for a given file ID. :param str file_id: a file ID """ pass b2-sdk-python-1.17.3/b2sdk/account_info/exception.py000066400000000000000000000025661426424117700222510ustar00rootroot00000000000000###################################################################### # # File: b2sdk/account_info/exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import (ABCMeta) from ..exception import B2Error class AccountInfoError(B2Error, metaclass=ABCMeta): """ Base class for all account info errors. """ pass class CorruptAccountInfo(AccountInfoError): """ Raised when an account info file is corrupted. """ def __init__(self, file_name): """ :param file_name: an account info file name :type file_name: str """ super(CorruptAccountInfo, self).__init__() self.file_name = file_name def __str__(self): return 'Account info file (%s) appears corrupted. Try removing and then re-authorizing the account.' % ( self.file_name, ) class MissingAccountData(AccountInfoError): """ Raised when there is no account info data available. """ def __init__(self, key): """ :param key: a key for getting account data :type key: str """ super(MissingAccountData, self).__init__() self.key = key def __str__(self): return 'Missing account data: %s' % (self.key,) b2-sdk-python-1.17.3/b2sdk/account_info/in_memory.py000066400000000000000000000103221426424117700222360ustar00rootroot00000000000000###################################################################### # # File: b2sdk/account_info/in_memory.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional from .exception import MissingAccountData from .upload_url_pool import UrlPoolAccountInfo from functools import wraps def _raise_missing_if_result_is_none(function): """ Raise MissingAccountData if function's result is None. """ @wraps(function) def inner(*args, **kwargs): assert function.__name__.startswith('get_') result = function(*args, **kwargs) if result is None: # assumes that it is a "get_field_name" raise MissingAccountData(function.__name__[4:]) return result return inner class InMemoryAccountInfo(UrlPoolAccountInfo): """ *AccountInfo* which keeps all data in memory. """ def __init__(self, *args, **kwargs): super(InMemoryAccountInfo, self).__init__(*args, **kwargs) self._clear_in_memory_account_fields() def clear(self): self._clear_in_memory_account_fields() return super(InMemoryAccountInfo, self).clear() def _clear_in_memory_account_fields(self): self._account_id = None self._application_key_id = None self._allowed = None self._api_url = None self._application_key = None self._auth_token = None self._buckets = {} self._download_url = None self._recommended_part_size = None self._absolute_minimum_part_size = None self._realm = None self._s3_api_url = None def _set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id ): self._account_id = account_id self._application_key_id = application_key_id self._auth_token = auth_token self._api_url = api_url self._download_url = download_url self._absolute_minimum_part_size = absolute_minimum_part_size self._recommended_part_size = recommended_part_size self._application_key = application_key self._realm = realm self._s3_api_url = s3_api_url self._allowed = allowed def refresh_entire_bucket_name_cache(self, name_id_iterable): self._buckets = dict(name_id_iterable) def get_bucket_id_or_none_from_bucket_name(self, bucket_name): return self._buckets.get(bucket_name) def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> Optional[str]: for name, cached_id_ in self._buckets.items(): if cached_id_ == bucket_id: return name return None def save_bucket(self, bucket): self._buckets[bucket.name] = bucket.id_ def remove_bucket_name(self, bucket_name): if bucket_name in self._buckets: del self._buckets[bucket_name] @_raise_missing_if_result_is_none def get_account_id(self): return self._account_id @_raise_missing_if_result_is_none def get_application_key_id(self): return self._application_key_id @_raise_missing_if_result_is_none def get_account_auth_token(self): return self._auth_token @_raise_missing_if_result_is_none def get_api_url(self): return self._api_url @_raise_missing_if_result_is_none def get_application_key(self): return self._application_key @_raise_missing_if_result_is_none def get_download_url(self): return self._download_url @_raise_missing_if_result_is_none def get_recommended_part_size(self): return self._recommended_part_size @_raise_missing_if_result_is_none def get_absolute_minimum_part_size(self): return self._absolute_minimum_part_size @_raise_missing_if_result_is_none def get_realm(self): return self._realm @_raise_missing_if_result_is_none def get_allowed(self): return self._allowed @_raise_missing_if_result_is_none def get_s3_api_url(self): return self._s3_api_url b2-sdk-python-1.17.3/b2sdk/account_info/sqlite_account_info.py000066400000000000000000000553011426424117700242760ustar00rootroot00000000000000###################################################################### # # File: b2sdk/account_info/sqlite_account_info.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import json import logging import os import re import sqlite3 import stat import threading from typing import List, Optional from .exception import CorruptAccountInfo, MissingAccountData from .upload_url_pool import UrlPoolAccountInfo logger = logging.getLogger(__name__) B2_ACCOUNT_INFO_ENV_VAR = 'B2_ACCOUNT_INFO' B2_ACCOUNT_INFO_DEFAULT_FILE = os.path.join('~', '.b2_account_info') B2_ACCOUNT_INFO_PROFILE_FILE = os.path.join('~', '.b2db-{profile}.sqlite') B2_ACCOUNT_INFO_PROFILE_NAME_REGEXP = re.compile(r'[a-zA-Z0-9_\-]{1,64}') XDG_CONFIG_HOME_ENV_VAR = 'XDG_CONFIG_HOME' DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE = 5000000 # this value is used ONLY in migrating db, and in v1 wrapper, it is not # meant to be a default for other applications class SqliteAccountInfo(UrlPoolAccountInfo): """ Store account information in an `sqlite3 `_ database which is used to manage concurrent access to the data. The ``update_done`` table tracks the schema updates that have been completed. """ def __init__(self, file_name=None, last_upgrade_to_run=None, profile: Optional[str] = None): """ Initialize SqliteAccountInfo. The exact algorithm used to determine the location of the database file is not API in any sense. If the location of the database file is required (for cleanup, etc), do not assume a specific resolution: instead, use ``self.filename`` to get the actual resolved location. SqliteAccountInfo currently checks locations in the following order: If ``profile`` arg is provided: * ``{XDG_CONFIG_HOME_ENV_VAR}/b2/db-.sqlite``, if ``{XDG_CONFIG_HOME_ENV_VAR}`` env var is set * ``{B2_ACCOUNT_INFO_PROFILE_FILE}`` Otherwise: * ``file_name``, if truthy * ``{B2_ACCOUNT_INFO_ENV_VAR}`` env var's value, if set * ``{B2_ACCOUNT_INFO_DEFAULT_FILE}``, if it exists * ``{XDG_CONFIG_HOME_ENV_VAR}/b2/account_info``, if ``{XDG_CONFIG_HOME_ENV_VAR}`` env var is set * ``{B2_ACCOUNT_INFO_DEFAULT_FILE}``, as default If the directory ``{XDG_CONFIG_HOME_ENV_VAR}/b2`` does not exist (and is needed), it is created. :param str file_name: The sqlite file to use; overrides the default. :param int last_upgrade_to_run: For testing only, override the auto-update on the db. """ self.thread_local = threading.local() self.filename = self._get_user_account_info_path(file_name=file_name, profile=profile) logger.debug('%s file path to use: %s', self.__class__.__name__, self.filename) self._validate_database(last_upgrade_to_run) with self._get_connection() as conn: self._create_tables(conn, last_upgrade_to_run) super(SqliteAccountInfo, self).__init__() # dirty trick to use parameters in the docstring if getattr(__init__, '__doc__', None): # don't break when using `python -oo` __init__.__doc__ = __init__.__doc__.format( **dict( B2_ACCOUNT_INFO_ENV_VAR=B2_ACCOUNT_INFO_ENV_VAR, B2_ACCOUNT_INFO_DEFAULT_FILE=B2_ACCOUNT_INFO_DEFAULT_FILE, B2_ACCOUNT_INFO_PROFILE_FILE=B2_ACCOUNT_INFO_PROFILE_FILE, XDG_CONFIG_HOME_ENV_VAR=XDG_CONFIG_HOME_ENV_VAR, ) ) @classmethod def _get_user_account_info_path(cls, file_name=None, profile=None): if profile and not B2_ACCOUNT_INFO_PROFILE_NAME_REGEXP.match(profile): raise ValueError('Invalid profile name: {}'.format(profile)) if file_name: if profile: raise ValueError('Provide either file_name or profile, not both') user_account_info_path = file_name elif B2_ACCOUNT_INFO_ENV_VAR in os.environ: if profile: raise ValueError( 'Provide either {} env var or profile, not both'. format(B2_ACCOUNT_INFO_ENV_VAR) ) user_account_info_path = os.environ[B2_ACCOUNT_INFO_ENV_VAR] elif os.path.exists(os.path.expanduser(B2_ACCOUNT_INFO_DEFAULT_FILE)) and not profile: user_account_info_path = B2_ACCOUNT_INFO_DEFAULT_FILE elif XDG_CONFIG_HOME_ENV_VAR in os.environ: config_home = os.environ[XDG_CONFIG_HOME_ENV_VAR] file_name = 'db-{}.sqlite'.format(profile) if profile else 'account_info' user_account_info_path = os.path.join(config_home, 'b2', file_name) if not os.path.exists(os.path.join(config_home, 'b2')): os.makedirs(os.path.join(config_home, 'b2'), mode=0o755) elif profile: user_account_info_path = B2_ACCOUNT_INFO_PROFILE_FILE.format(profile=profile) else: user_account_info_path = B2_ACCOUNT_INFO_DEFAULT_FILE return os.path.expanduser(user_account_info_path) def _validate_database(self, last_upgrade_to_run=None): """ Make sure that the database is openable. Removes the file if it's not. """ # If there is no file there, that's fine. It will get created when # we connect. if not os.path.exists(self.filename): self._create_database(last_upgrade_to_run) return # If we can connect to the database, and do anything, then all is good. try: with self._connect() as conn: self._create_tables(conn, last_upgrade_to_run) return except sqlite3.DatabaseError: pass # fall through to next case # If the file contains JSON with the right stuff in it, convert from # the old representation. try: with open(self.filename, 'rb') as f: data = json.loads(f.read().decode('utf-8')) keys = [ 'account_id', 'application_key', 'account_auth_token', 'api_url', 'download_url', 'minimum_part_size', 'realm' ] if all(k in data for k in keys): # remove the json file os.unlink(self.filename) # create a database self._create_database(last_upgrade_to_run) # add the data from the JSON file with self._connect() as conn: self._create_tables(conn, last_upgrade_to_run) insert_statement = """ INSERT INTO account (account_id, application_key, account_auth_token, api_url, download_url, recommended_part_size, realm, absolute_minimum_part_size) values (?, ?, ?, ?, ?, ?, ?, ?); """ # Migrating from old schema is a little confusing, but the values change as: # minimum_part_size -> recommended_part_size # new column absolute_minimum_part_size = DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE conn.execute( insert_statement, (*(data[k] for k in keys), DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE) ) # all is happy now return except ValueError: # includes json.decoder.JSONDecodeError pass # Remove the corrupted file and create a new database raise CorruptAccountInfo(self.filename) def _get_connection(self): """ Connections to sqlite cannot be shared across threads. """ try: return self.thread_local.connection except AttributeError: self.thread_local.connection = self._connect() return self.thread_local.connection def _connect(self): return sqlite3.connect(self.filename, isolation_level='EXCLUSIVE') def _create_database(self, last_upgrade_to_run): """ Make sure that the database is created and has appropriate file permissions. This should be done before storing any sensitive data in it. """ # Prepare a file fd = os.open( self.filename, flags=os.O_RDWR | os.O_CREAT, mode=stat.S_IRUSR | stat.S_IWUSR, ) os.close(fd) # Create the tables in the database conn = self._connect() try: with conn: self._create_tables(conn, last_upgrade_to_run) finally: conn.close() def _create_tables(self, conn, last_upgrade_to_run): conn.execute( """ CREATE TABLE IF NOT EXISTS update_done ( update_number INT NOT NULL ); """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS account ( account_id TEXT NOT NULL, application_key TEXT NOT NULL, account_auth_token TEXT NOT NULL, api_url TEXT NOT NULL, download_url TEXT NOT NULL, minimum_part_size INT NOT NULL, realm TEXT NOT NULL ); """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS bucket ( bucket_name TEXT NOT NULL, bucket_id TEXT NOT NULL ); """ ) # This table is not used any more. We may use it again # someday if we save upload URLs across invocations of # the command-line tool. conn.execute( """ CREATE TABLE IF NOT EXISTS bucket_upload_url ( bucket_id TEXT NOT NULL, upload_url TEXT NOT NULL, upload_auth_token TEXT NOT NULL ); """ ) # By default, we run all the upgrades last_upgrade_to_run = 4 if last_upgrade_to_run is None else last_upgrade_to_run # Add the 'allowed' column if it hasn't been yet. if 1 <= last_upgrade_to_run: self._ensure_update(1, ['ALTER TABLE account ADD COLUMN allowed TEXT;']) # Add the 'account_id_or_app_key_id' column if it hasn't been yet if 2 <= last_upgrade_to_run: self._ensure_update( 2, ['ALTER TABLE account ADD COLUMN account_id_or_app_key_id TEXT;'] ) # Add the 's3_api_url' column if it hasn't been yet if 3 <= last_upgrade_to_run: self._ensure_update(3, ['ALTER TABLE account ADD COLUMN s3_api_url TEXT;']) if 4 <= last_upgrade_to_run: self._ensure_update( 4, [ """ CREATE TABLE tmp_account ( account_id TEXT NOT NULL, application_key TEXT NOT NULL, account_auth_token TEXT NOT NULL, api_url TEXT NOT NULL, download_url TEXT NOT NULL, absolute_minimum_part_size INT NOT NULL DEFAULT {}, recommended_part_size INT NOT NULL, realm TEXT NOT NULL, allowed TEXT, account_id_or_app_key_id TEXT, s3_api_url TEXT ); """.format(DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE), """INSERT INTO tmp_account( account_id, application_key, account_auth_token, api_url, download_url, recommended_part_size, realm, allowed, account_id_or_app_key_id, s3_api_url ) SELECT account_id, application_key, account_auth_token, api_url, download_url, minimum_part_size, realm, allowed, account_id_or_app_key_id, s3_api_url FROM account; """, 'DROP TABLE account;', """ CREATE TABLE account ( account_id TEXT NOT NULL, application_key TEXT NOT NULL, account_auth_token TEXT NOT NULL, api_url TEXT NOT NULL, download_url TEXT NOT NULL, absolute_minimum_part_size INT NOT NULL, recommended_part_size INT NOT NULL, realm TEXT NOT NULL, allowed TEXT, account_id_or_app_key_id TEXT, s3_api_url TEXT ); """, """INSERT INTO account( account_id, application_key, account_auth_token, api_url, download_url, absolute_minimum_part_size, recommended_part_size, realm, allowed, account_id_or_app_key_id, s3_api_url ) SELECT account_id, application_key, account_auth_token, api_url, download_url, absolute_minimum_part_size, recommended_part_size, realm, allowed, account_id_or_app_key_id, s3_api_url FROM tmp_account; """, 'DROP TABLE tmp_account;', ] ) def _ensure_update(self, update_number, update_commands: List[str]): """ Run the update with the given number if it hasn't been done yet. Does the update and stores the number as a single transaction, so they will always be in sync. """ with self._get_connection() as conn: conn.execute('BEGIN') cursor = conn.execute( 'SELECT COUNT(*) AS count FROM update_done WHERE update_number = ?;', (update_number,) ) update_count = cursor.fetchone()[0] if update_count == 0: for command in update_commands: conn.execute(command) conn.execute( 'INSERT INTO update_done (update_number) VALUES (?);', (update_number,) ) def clear(self): """ Remove all info about accounts and buckets. """ with self._get_connection() as conn: conn.execute('DELETE FROM account;') conn.execute('DELETE FROM bucket;') conn.execute('DELETE FROM bucket_upload_url;') def _set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id, ): assert self.allowed_is_valid(allowed) with self._get_connection() as conn: conn.execute('DELETE FROM account;') conn.execute('DELETE FROM bucket;') conn.execute('DELETE FROM bucket_upload_url;') insert_statement = """ INSERT INTO account (account_id, account_id_or_app_key_id, application_key, account_auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, realm, allowed, s3_api_url) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ conn.execute( insert_statement, ( account_id, application_key_id, application_key, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, realm, json.dumps(allowed), s3_api_url, ) ) def set_auth_data_with_schema_0_for_test( self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, ): """ Set authentication data for tests. :param str account_id: an account ID :param str auth_token: an authentication token :param str api_url: an API URL :param str download_url: a download URL :param int minimum_part_size: a minimum part size :param str application_key: an application key :param str realm: a realm to authorize account in """ with self._get_connection() as conn: conn.execute('DELETE FROM account;') conn.execute('DELETE FROM bucket;') conn.execute('DELETE FROM bucket_upload_url;') insert_statement = """ INSERT INTO account (account_id, application_key, account_auth_token, api_url, download_url, minimum_part_size, realm) values (?, ?, ?, ?, ?, ?, ?); """ conn.execute( insert_statement, ( account_id, application_key, auth_token, api_url, download_url, minimum_part_size, realm, ) ) def get_application_key(self): return self._get_account_info_or_raise('application_key') def get_account_id(self): return self._get_account_info_or_raise('account_id') def get_application_key_id(self): """ Return an application key ID. The 'account_id_or_app_key_id' column was not in the original schema, so it may be NULL. Nota bene - this is the only place where we are not renaming account_id_or_app_key_id to application_key_id because it requires a column change. application_key_id == account_id_or_app_key_id :rtype: str """ result = self._get_account_info_or_raise('account_id_or_app_key_id') if result is None: return self.get_account_id() else: return result def get_api_url(self): return self._get_account_info_or_raise('api_url') def get_account_auth_token(self): return self._get_account_info_or_raise('account_auth_token') def get_download_url(self): return self._get_account_info_or_raise('download_url') def get_realm(self): return self._get_account_info_or_raise('realm') def get_recommended_part_size(self): return self._get_account_info_or_raise('recommended_part_size') def get_absolute_minimum_part_size(self): return self._get_account_info_or_raise('absolute_minimum_part_size') def get_allowed(self): """ Return 'allowed' dictionary info. Example: .. code-block:: python { "bucketId": null, "bucketName": null, "capabilities": [ "listKeys", "writeKeys" ], "namePrefix": null } The 'allowed' column was not in the original schema, so it may be NULL. :rtype: dict """ allowed_json = self._get_account_info_or_raise('allowed') if allowed_json is None: return self.DEFAULT_ALLOWED else: return json.loads(allowed_json) def get_s3_api_url(self): result = self._get_account_info_or_raise('s3_api_url') if result is None: return '' else: return result def _get_account_info_or_raise(self, column_name): try: with self._get_connection() as conn: cursor = conn.execute('SELECT %s FROM account;' % (column_name,)) value = cursor.fetchone()[0] return value except Exception as e: logger.exception( '_get_account_info_or_raise encountered a problem while trying to retrieve "%s"', column_name ) raise MissingAccountData(str(e)) def refresh_entire_bucket_name_cache(self, name_id_iterable): with self._get_connection() as conn: conn.execute('DELETE FROM bucket;') for (bucket_name, bucket_id) in name_id_iterable: conn.execute( 'INSERT INTO bucket (bucket_name, bucket_id) VALUES (?, ?);', (bucket_name, bucket_id) ) def save_bucket(self, bucket): with self._get_connection() as conn: conn.execute('DELETE FROM bucket WHERE bucket_id = ?;', (bucket.id_,)) conn.execute( 'INSERT INTO bucket (bucket_id, bucket_name) VALUES (?, ?);', (bucket.id_, bucket.name) ) def remove_bucket_name(self, bucket_name): with self._get_connection() as conn: conn.execute('DELETE FROM bucket WHERE bucket_name = ?;', (bucket_name,)) def get_bucket_id_or_none_from_bucket_name(self, bucket_name): return self._safe_query( 'SELECT bucket_id FROM bucket WHERE bucket_name = ?;', (bucket_name,) ) def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> Optional[str]: return self._safe_query('SELECT bucket_name FROM bucket WHERE bucket_id = ?;', (bucket_id,)) def _safe_query(self, query, params): try: with self._get_connection() as conn: cursor = conn.execute(query, params) return cursor.fetchone()[0] except TypeError: # TypeError: 'NoneType' object is unsubscriptable return None except sqlite3.Error: return None b2-sdk-python-1.17.3/b2sdk/account_info/stub.py000066400000000000000000000101201426424117700212110ustar00rootroot00000000000000###################################################################### # # File: b2sdk/account_info/stub.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional import collections import threading from .abstract import AbstractAccountInfo class StubAccountInfo(AbstractAccountInfo): def __init__(self): self._clear_stub_account_fields() def clear(self): self._clear_stub_account_fields() def _clear_stub_account_fields(self): self.application_key = None self.buckets = {} self.account_id = None self.allowed = None self.api_url = None self.auth_token = None self.download_url = None self.absolute_minimum_part_size = None self.recommended_part_size = None self.realm = None self.application_key_id = None self._large_file_uploads = collections.defaultdict(list) self._large_file_uploads_lock = threading.Lock() def clear_bucket_upload_data(self, bucket_id): if bucket_id in self.buckets: del self.buckets[bucket_id] def _set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id, ): self.account_id = account_id self.auth_token = auth_token self.api_url = api_url self.download_url = download_url self.recommended_part_size = recommended_part_size self.absolute_minimum_part_size = absolute_minimum_part_size self.application_key = application_key self.realm = realm self.s3_api_url = s3_api_url self.allowed = allowed self.application_key_id = application_key_id def refresh_entire_bucket_name_cache(self, name_id_iterable): self.buckets = {} def get_bucket_id_or_none_from_bucket_name(self, bucket_name): return None def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> Optional[str]: return None def save_bucket(self, bucket): self.buckets[bucket.bucket_id] = bucket def remove_bucket_name(self, bucket_name): pass def take_bucket_upload_url(self, bucket_id): return (None, None) def put_bucket_upload_url(self, bucket_id, upload_url, upload_auth_token): pass def get_account_id(self): return self.account_id def get_application_key_id(self): return self.application_key_id def get_account_auth_token(self): return self.auth_token def get_api_url(self): return self.api_url def get_application_key(self): return self.application_key def get_download_url(self): return self.download_url def get_recommended_part_size(self): return self.recommended_part_size def get_absolute_minimum_part_size(self): return self.absolute_minimum_part_size def get_realm(self): return self.realm def get_allowed(self): return self.allowed def get_bucket_upload_data(self, bucket_id): return self.buckets.get(bucket_id, (None, None)) def get_s3_api_url(self): return self.s3_api_url def put_large_file_upload_url(self, file_id, upload_url, upload_auth_token): with self._large_file_uploads_lock: self._large_file_uploads[file_id].append((upload_url, upload_auth_token)) def take_large_file_upload_url(self, file_id): with self._large_file_uploads_lock: upload_urls = self._large_file_uploads.get(file_id, []) if not upload_urls: return (None, None) else: return upload_urls.pop() def clear_large_file_upload_urls(self, file_id): with self._large_file_uploads_lock: if file_id in self._large_file_uploads: del self._large_file_uploads[file_id] b2-sdk-python-1.17.3/b2sdk/account_info/test_upload_url_concurrency.py000066400000000000000000000031531426424117700260630ustar00rootroot00000000000000###################################################################### # # File: b2sdk/account_info/test_upload_url_concurrency.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os import threading from .sqlite_account_info import SqliteAccountInfo def test_upload_url_concurrency(): # Clean up from previous tests file_name = '/tmp/test_upload_conncurrency.db' try: os.unlink(file_name) except OSError: pass # Make an account info with a bunch of upload URLs in it. account_info = SqliteAccountInfo(file_name) available_urls = set() for i in range(3000): url = 'url_%d' % i account_info.put_bucket_upload_url('bucket-id', url, 'auth-token-%d' % i) available_urls.add(url) # Pull them all from the account info, from multiple threads lock = threading.Lock() def run_thread(): while True: (url, _) = account_info.take_bucket_upload_url('bucket-id') if url is None: break with lock: if url in available_urls: available_urls.remove(url) else: print('DOUBLE:', url) threads = [] for i in range(5): thread = threading.Thread(target=run_thread) thread.start() threads.append(thread) for t in threads: t.join() # Check if len(available_urls) != 0: print('LEAK:', available_urls) # Clean up os.unlink(file_name) b2-sdk-python-1.17.3/b2sdk/account_info/upload_url_pool.py000066400000000000000000000067211426424117700234470ustar00rootroot00000000000000###################################################################### # # File: b2sdk/account_info/upload_url_pool.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import abstractmethod import collections import threading from .abstract import AbstractAccountInfo class UploadUrlPool: """ For each key (either a bucket id or large file id), hold a pool of (url, auth_token) pairs. .. note: This class is thread-safe. """ def __init__(self): self._lock = threading.Lock() self._pool = collections.defaultdict(list) def put(self, key, url, auth_token): """ Add the url and auth token to the pool for the given key. :param str key: bucket ID or large file ID :param str url: bucket or file URL :param str auth_token: authentication token """ with self._lock: pair = (url, auth_token) self._pool[key].append(pair) def take(self, key): """ Return a (url, auth_token) if one is available, or (None, None) if not. :param str key: bucket ID or large file ID :rtype: tuple """ with self._lock: pair_list = self._pool[key] if pair_list: return pair_list.pop() else: return (None, None) def clear_for_key(self, key): """ Remove an item from the pool by key. :param str key: bucket ID or large file ID """ with self._lock: if key in self._pool: del self._pool[key] class UrlPoolAccountInfo(AbstractAccountInfo): """ Implement part of :py:class:`AbstractAccountInfo` for upload URL pool management with a simple, key-value storage, such as :py:class:`b2sdk.v2.UploadUrlPool`. """ # staticmethod is necessary here to avoid the first argument binding to the first argument (like ``partial(fun, arg)``) BUCKET_UPLOAD_POOL_CLASS = staticmethod( UploadUrlPool ) #: A url pool class to use for small files. LARGE_FILE_UPLOAD_POOL_CLASS = staticmethod( UploadUrlPool ) #: A url pool class to use for large files. def __init__(self): super(UrlPoolAccountInfo, self).__init__() self._reset_upload_pools() @abstractmethod def clear(self): self._reset_upload_pools() return super(UrlPoolAccountInfo, self).clear() def _reset_upload_pools(self): self._bucket_uploads = self.BUCKET_UPLOAD_POOL_CLASS() self._large_file_uploads = self.LARGE_FILE_UPLOAD_POOL_CLASS() # bucket upload url def put_bucket_upload_url(self, bucket_id, upload_url, upload_auth_token): self._bucket_uploads.put(bucket_id, upload_url, upload_auth_token) def clear_bucket_upload_data(self, bucket_id): self._bucket_uploads.clear_for_key(bucket_id) def take_bucket_upload_url(self, bucket_id): return self._bucket_uploads.take(bucket_id) # large file upload url def put_large_file_upload_url(self, file_id, upload_url, upload_auth_token): self._large_file_uploads.put(file_id, upload_url, upload_auth_token) def take_large_file_upload_url(self, file_id): return self._large_file_uploads.take(file_id) def clear_large_file_upload_urls(self, file_id): self._large_file_uploads.clear_for_key(file_id) b2-sdk-python-1.17.3/b2sdk/api.py000066400000000000000000000545031426424117700163530ustar00rootroot00000000000000###################################################################### # # File: b2sdk/api.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional, Tuple, List, Generator from .account_info.abstract import AbstractAccountInfo from .api_config import B2HttpApiConfig, DEFAULT_HTTP_API_CONFIG from .application_key import ApplicationKey, BaseApplicationKey, FullApplicationKey from .cache import AbstractCache from .bucket import Bucket, BucketFactory from .encryption.setting import EncryptionSetting from .replication.setting import ReplicationConfiguration from .exception import BucketIdNotFound, NonExistentBucket, RestrictedBucket from .file_lock import FileRetentionSetting, LegalHold from .file_version import DownloadVersionFactory, FileIdAndName, FileVersion, FileVersionFactory from .large_file.services import LargeFileServices from .raw_api import API_VERSION from .progress import AbstractProgressListener from .session import B2Session from .transfer import ( CopyManager, DownloadManager, Emerger, UploadManager, ) from .transfer.inbound.downloaded_file import DownloadedFile from .utils import B2TraceMeta, b2_url_encode, limit_trace_arguments def url_for_api(info, api_name): """ Return URL for an API endpoint. :param info: account info :param str api_nam: :rtype: str """ if api_name in ['b2_download_file_by_id']: base = info.get_download_url() else: base = info.get_api_url() return '%s/b2api/%s/%s' % (base, API_VERSION, api_name) class Services: """ Gathers objects that provide high level logic over raw api usage. """ UPLOAD_MANAGER_CLASS = staticmethod(UploadManager) COPY_MANAGER_CLASS = staticmethod(CopyManager) DOWNLOAD_MANAGER_CLASS = staticmethod(DownloadManager) def __init__( self, api, max_upload_workers: Optional[int] = None, max_copy_workers: Optional[int] = None, max_download_workers: Optional[int] = None, save_to_buffer_size: Optional[int] = None, check_download_hash: bool = True, ): """ Initialize Services object using given session. :param b2sdk.v2.B2Api api: :param max_upload_workers: a number of upload threads :param max_copy_workers: a number of copy threads :param max_download_workers: maximum number of download threads :param save_to_buffer_size: buffer size to use when writing files using DownloadedFile.save_to :param check_download_hash: whether to check hash of downloaded files. Can be disabled for files with internal checksums, for example, or to forcefully retrieve objects with corrupted payload or hash value """ self.api = api self.session = api.session self.large_file = LargeFileServices(self) self.upload_manager = self.UPLOAD_MANAGER_CLASS( services=self, max_workers=max_upload_workers ) self.copy_manager = self.COPY_MANAGER_CLASS(services=self, max_workers=max_copy_workers) self.download_manager = self.DOWNLOAD_MANAGER_CLASS( services=self, max_workers=max_download_workers, write_buffer_size=save_to_buffer_size, check_hash=check_download_hash, ) self.emerger = Emerger(self) class B2Api(metaclass=B2TraceMeta): """ Provide file-level access to B2 services. While :class:`b2sdk.v2.B2RawHTTPApi` provides direct access to the B2 web APIs, this class handles several things that simplify the task of uploading and downloading files: - re-acquires authorization tokens when they expire - retrying uploads when an upload URL is busy - breaking large files into parts - emulating a directory structure (B2 buckets are flat) Adds an object-oriented layer on top of the raw API, so that buckets and files returned are Python objects with accessor methods. The class also keeps a cache of information needed to access the service, such as auth tokens and upload URLs. """ BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) BUCKET_CLASS = staticmethod(Bucket) SESSION_CLASS = staticmethod(B2Session) FILE_VERSION_FACTORY_CLASS = staticmethod(FileVersionFactory) DOWNLOAD_VERSION_FACTORY_CLASS = staticmethod(DownloadVersionFactory) SERVICES_CLASS = staticmethod(Services) DEFAULT_LIST_KEY_COUNT = 1000 def __init__( self, account_info: Optional[AbstractAccountInfo] = None, cache: Optional[AbstractCache] = None, max_upload_workers: Optional[int] = None, max_copy_workers: Optional[int] = None, api_config: B2HttpApiConfig = DEFAULT_HTTP_API_CONFIG, max_download_workers: Optional[int] = None, save_to_buffer_size: Optional[int] = None, check_download_hash: bool = True, ): """ Initialize the API using the given account info. :param account_info: To learn more about Account Info objects, see here :class:`~b2sdk.v2.SqliteAccountInfo` :param cache: It is used by B2Api to cache the mapping between bucket name and bucket ids. default is :class:`~b2sdk.cache.DummyCache` :param max_upload_workers: a number of upload threads :param max_copy_workers: a number of copy threads :param api_config: :param max_download_workers: maximum number of download threads :param save_to_buffer_size: buffer size to use when writing files using DownloadedFile.save_to :param check_download_hash: whether to check hash of downloaded files. Can be disabled for files with internal checksums, for example, or to forcefully retrieve objects with corrupted payload or hash value """ self.session = self.SESSION_CLASS( account_info=account_info, cache=cache, api_config=api_config ) self.api_config = api_config self.file_version_factory = self.FILE_VERSION_FACTORY_CLASS(self) self.download_version_factory = self.DOWNLOAD_VERSION_FACTORY_CLASS(self) self.services = self.SERVICES_CLASS( api=self, max_upload_workers=max_upload_workers, max_copy_workers=max_copy_workers, max_download_workers=max_download_workers, save_to_buffer_size=save_to_buffer_size, check_download_hash=check_download_hash, ) @property def account_info(self): return self.session.account_info @property def cache(self): return self.session.cache @property def raw_api(self): """ .. warning:: :class:`~b2sdk.raw_api.B2RawHTTPApi` attribute is deprecated. :class:`~b2sdk.session.B2Session` expose all :class:`~b2sdk.raw_api.B2RawHTTPApi` methods now.""" return self.session.raw_api def authorize_automatically(self): """ Perform automatic account authorization, retrieving all account data from account info object passed during initialization. """ return self.session.authorize_automatically() @limit_trace_arguments(only=('self', 'realm')) def authorize_account(self, realm, application_key_id, application_key): """ Perform account authorization. :param str realm: a realm to authorize account in (usually just "production") :param str application_key_id: :term:`application key ID` :param str application_key: user's :term:`application key` """ self.session.authorize_account(realm, application_key_id, application_key) def get_account_id(self): """ Return the account ID. :rtype: str """ return self.account_info.get_account_id() # buckets def create_bucket( self, name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, is_file_lock_enabled: Optional[bool] = None, replication: Optional[ReplicationConfiguration] = None, ): """ Create a bucket. :param str name: bucket name :param str bucket_type: a bucket type, could be one of the following values: ``"allPublic"``, ``"allPrivate"`` :param dict bucket_info: additional bucket info to store with the bucket :param dict cors_rules: bucket CORS rules to store with the bucket :param list lifecycle_rules: bucket lifecycle rules to store with the bucket :param b2sdk.v2.EncryptionSetting default_server_side_encryption: default server side encryption settings (``None`` if unknown) :param bool is_file_lock_enabled: boolean value specifies whether bucket is File Lock-enabled :param b2sdk.v2.ReplicationConfiguration replication: bucket replication rules or ``None`` :return: a Bucket object :rtype: b2sdk.v2.Bucket """ account_id = self.account_info.get_account_id() response = self.session.create_bucket( account_id, name, bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, default_server_side_encryption=default_server_side_encryption, is_file_lock_enabled=is_file_lock_enabled, replication=replication, ) bucket = self.BUCKET_FACTORY_CLASS.from_api_bucket_dict(self, response) assert name == bucket.name, 'API created a bucket with different name\ than requested: %s != %s' % (name, bucket.name) assert bucket_type == bucket.type_, 'API created a bucket with different type\ than requested: %s != %s' % ( bucket_type, bucket.type_ ) self.cache.save_bucket(bucket) return bucket def download_file_by_id( self, file_id: str, progress_listener: Optional[AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[EncryptionSetting] = None, ) -> DownloadedFile: """ Download a file with the given ID. :param str file_id: a file ID :param progress_listener: a progress listener object to use, or ``None`` to not track progress :param range_: a list of two integers, the first one is a start\ position, and the second one is the end position in the file :param encryption: encryption settings (``None`` if unknown) """ url = self.session.get_download_url_by_id(file_id) return self.services.download_manager.download_file_from_url( url, progress_listener, range_, encryption, ) def update_file_retention( self, file_id: str, file_name: str, file_retention: FileRetentionSetting, bypass_governance: bool = False, ) -> FileRetentionSetting: return FileRetentionSetting.from_server_response( self.session.update_file_retention( file_id, file_name, file_retention, bypass_governance, ) ) def update_file_legal_hold( self, file_id: str, file_name: str, legal_hold: LegalHold, ) -> LegalHold: return LegalHold.from_server_response( self.session.update_file_legal_hold( file_id, file_name, legal_hold, ) ) def get_bucket_by_id(self, bucket_id: str) -> Bucket: """ Return the Bucket matching the given bucket_id. :raises b2sdk.v2.exception.BucketIdNotFound: if the bucket does not exist in the account """ # Give a useful warning if the current application key does not # allow access to bucket. self.check_bucket_id_restrictions(bucket_id) # First, try the cache. bucket_name = self.cache.get_bucket_name_or_none_from_bucket_id(bucket_id) if bucket_name is not None: return self.BUCKET_CLASS(self, bucket_id, name=bucket_name) # Second, ask the service for bucket in self.list_buckets(bucket_id=bucket_id): assert bucket.id_ == bucket_id return bucket # There is no such bucket. raise BucketIdNotFound(bucket_id) def get_bucket_by_name(self, bucket_name: str): """ Return the Bucket matching the given bucket_name. :param str bucket_name: the name of the bucket to return :return: a Bucket object :rtype: b2sdk.v2.Bucket :raises b2sdk.v2.exception.NonExistentBucket: if the bucket does not exist in the account """ # Give a useful warning if the current application key does not # allow access to the named bucket. self.check_bucket_name_restrictions(bucket_name) # First, try the cache. id_ = self.cache.get_bucket_id_or_none_from_bucket_name(bucket_name) if id_ is not None: return self.BUCKET_CLASS(self, id_, name=bucket_name) # Second, ask the service for bucket in self.list_buckets(bucket_name=bucket_name): assert bucket.name.lower() == bucket_name.lower() return bucket # There is no such bucket. raise NonExistentBucket(bucket_name) def delete_bucket(self, bucket): """ Delete a chosen bucket. :param b2sdk.v2.Bucket bucket: a :term:`bucket` to delete :rtype: None """ account_id = self.account_info.get_account_id() self.session.delete_bucket(account_id, bucket.id_) def list_buckets(self, bucket_name=None, bucket_id=None): """ Call ``b2_list_buckets`` and return a list of buckets. When no bucket name nor ID is specified, returns *all* of the buckets in the account. When a bucket name or ID is given, returns just that bucket. When authorized with an :term:`application key` restricted to one :term:`bucket`, you must specify the bucket name or bucket id, or the request will be unauthorized. :param str bucket_name: the name of the one bucket to return :param str bucket_id: the ID of the one bucket to return :rtype: list[b2sdk.v2.Bucket] """ # Give a useful warning if the current application key does not # allow access to the named bucket. if bucket_name is not None and bucket_id is not None: raise ValueError('Provide either bucket_name or bucket_id, not both') if bucket_id: self.check_bucket_id_restrictions(bucket_id) else: self.check_bucket_name_restrictions(bucket_name) account_id = self.account_info.get_account_id() response = self.session.list_buckets( account_id, bucket_name=bucket_name, bucket_id=bucket_id ) buckets = self.BUCKET_FACTORY_CLASS.from_api_response(self, response) if bucket_name or bucket_id: # If a bucket_name or bucket_id is specified we don't clear the cache because the other buckets could still # be valid. So we save the one bucket returned from the list_buckets call. for bucket in buckets: self.cache.save_bucket(bucket) else: # Otherwise we want to clear the cache and save the buckets returned from list_buckets # since we just got a new list of all the buckets for this account. self.cache.set_bucket_name_cache(buckets) return buckets def list_parts(self, file_id, start_part_number=None, batch_size=None): """ Generator that yields a :py:class:`b2sdk.v2.Part` for each of the parts that have been uploaded. :param str file_id: the ID of the large file that is not finished :param int start_part_number: the first part number to return; defaults to the first part :param int batch_size: the number of parts to fetch at a time from the server :rtype: generator """ return self.services.large_file.list_parts( file_id, start_part_number=start_part_number, batch_size=batch_size ) # delete/cancel def cancel_large_file(self, file_id: str) -> FileIdAndName: """ Cancel a large file upload. """ return self.services.large_file.cancel_large_file(file_id) def delete_file_version(self, file_id: str, file_name: str) -> FileIdAndName: """ Permanently and irrevocably delete one version of a file. """ # filename argument is not first, because one day it may become optional response = self.session.delete_file_version(file_id, file_name) return FileIdAndName.from_cancel_or_delete_response(response) # download def get_download_url_for_fileid(self, file_id): """ Return a URL to download the given file by ID. :param str file_id: a file ID """ url = url_for_api(self.account_info, 'b2_download_file_by_id') return '%s?fileId=%s' % (url, file_id) def get_download_url_for_file_name(self, bucket_name, file_name): """ Return a URL to download the given file by name. :param str bucket_name: a bucket name :param str file_name: a file name """ self.check_bucket_name_restrictions(bucket_name) return '%s/file/%s/%s' % ( self.account_info.get_download_url(), bucket_name, b2_url_encode(file_name) ) # keys def create_key( self, capabilities: List[str], key_name: str, valid_duration_seconds: Optional[int] = None, bucket_id: Optional[str] = None, name_prefix: Optional[str] = None, ): """ Create a new :term:`application key`. :param capabilities: a list of capabilities :param key_name: a name of a key :param valid_duration_seconds: key auto-expire time after it is created, in seconds, or ``None`` to not expire :param bucket_id: a bucket ID to restrict the key to, or ``None`` to not restrict :param name_prefix: a remote filename prefix to restrict the key to or ``None`` to not restrict """ account_id = self.account_info.get_account_id() response = self.session.create_key( account_id, capabilities=capabilities, key_name=key_name, valid_duration_seconds=valid_duration_seconds, bucket_id=bucket_id, name_prefix=name_prefix ) assert set(response['capabilities']) == set(capabilities) assert response['keyName'] == key_name return FullApplicationKey.from_create_response(response) def delete_key(self, application_key: BaseApplicationKey): """ Delete :term:`application key`. :param application_key: an :term:`application key` """ return self.delete_key_by_id(application_key.id_) def delete_key_by_id(self, application_key_id: str): """ Delete :term:`application key`. :param application_key_id: an :term:`application key ID` """ response = self.session.delete_key(application_key_id=application_key_id) return ApplicationKey.from_api_response(response) def list_keys(self, start_application_key_id: Optional[str] = None ) -> Generator[ApplicationKey, None, None]: """ List application keys. Lazily perform requests to B2 cloud and return all keys. :param start_application_key_id: an :term:`application key ID` to start from or ``None`` to start from the beginning """ account_id = self.account_info.get_account_id() while True: response = self.session.list_keys( account_id, max_key_count=self.DEFAULT_LIST_KEY_COUNT, start_application_key_id=start_application_key_id, ) for entry in response['keys']: yield ApplicationKey.from_api_response(entry) next_application_key_id = response['nextApplicationKeyId'] if next_application_key_id is None: return start_application_key_id = next_application_key_id def get_key(self, key_id: str) -> Optional[ApplicationKey]: """ Gets information about a single key: it's capabilities, prefix, name etc Returns `None` if the key does not exist. Raises an exception if profile is not permitted to list keys. """ return next( self.list_keys(start_application_key_id=key_id), None, ) # other def get_file_info(self, file_id: str) -> FileVersion: """ Gets info about file version. :param str file_id: the id of the file whose info will be retrieved. """ return self.file_version_factory.from_api_response( self.session.get_file_info_by_id(file_id) ) def check_bucket_name_restrictions(self, bucket_name: str): """ Check to see if the allowed field from authorize-account has a bucket restriction. If it does, checks if the bucket_name for a given api call matches that. If not, it raises a :py:exc:`b2sdk.v2.exception.RestrictedBucket` error. :raises b2sdk.v2.exception.RestrictedBucket: if the account is not allowed to use this bucket """ self._check_bucket_restrictions('bucketName', bucket_name) def check_bucket_id_restrictions(self, bucket_id: str): """ Check to see if the allowed field from authorize-account has a bucket restriction. If it does, checks if the bucket_id for a given api call matches that. If not, it raises a :py:exc:`b2sdk.v2.exception.RestrictedBucket` error. :raises b2sdk.v2.exception.RestrictedBucket: if the account is not allowed to use this bucket """ self._check_bucket_restrictions('bucketId', bucket_id) def _check_bucket_restrictions(self, key, value): allowed = self.account_info.get_allowed() allowed_bucket_identifier = allowed[key] if allowed_bucket_identifier is not None: if allowed_bucket_identifier != value: raise RestrictedBucket(allowed_bucket_identifier) b2-sdk-python-1.17.3/b2sdk/api_config.py000066400000000000000000000032741426424117700176770ustar00rootroot00000000000000###################################################################### # # File: b2sdk/api_config.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional, Callable, Type import requests from .raw_api import AbstractRawApi, B2RawHTTPApi class B2HttpApiConfig: DEFAULT_RAW_API_CLASS = B2RawHTTPApi def __init__( self, http_session_factory: Callable[[], requests.Session] = requests.Session, install_clock_skew_hook: bool = True, user_agent_append: Optional[str] = None, _raw_api_class: Optional[Type[AbstractRawApi]] = None, decode_content: bool = False ): """ A structure with params to be passed to low level API. :param http_session_factory: a callable that returns a requests.Session object (or a compatible one) :param install_clock_skew_hook: if True, install a clock skew hook :param user_agent_append: if provided, the string will be appended to the User-Agent :param _raw_api_class: AbstractRawApi-compliant class :param decode_content: If true, the underlying http backend will try to decode encoded files when downloading, based on the response headers """ self.http_session_factory = http_session_factory self.install_clock_skew_hook = install_clock_skew_hook self.user_agent_append = user_agent_append self.raw_api_class = _raw_api_class or self.DEFAULT_RAW_API_CLASS self.decode_content = decode_content DEFAULT_HTTP_API_CONFIG = B2HttpApiConfig() b2-sdk-python-1.17.3/b2sdk/application_key.py000066400000000000000000000133351426424117700207530ustar00rootroot00000000000000###################################################################### # # File: b2sdk/application_key.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import List, Optional class BaseApplicationKey: """Common methods for ApplicationKey and FullApplicationKey.""" def __init__( self, key_name: str, application_key_id: str, capabilities: List[str], account_id: str, expiration_timestamp_millis: Optional[int] = None, bucket_id: Optional[str] = None, name_prefix: Optional[str] = None, options: Optional[List[str]] = None, ): """ :param key_name: name of the key, assigned by user :param application_key_id: key id, used to authenticate :param capabilities: list of capabilities assigned to this key :param account_id: account's id :param expiration_timestamp_millis: expiration time of the key :param bucket_id: if restricted to a bucket, this is the bucket's id :param name_prefix: if restricted to some files, this is their prefix :param options: reserved for future use """ self.key_name = key_name self.id_ = application_key_id self.capabilities = capabilities self.account_id = account_id self.expiration_timestamp_millis = expiration_timestamp_millis self.bucket_id = bucket_id self.name_prefix = name_prefix self.options = options @classmethod def parse_response_dict(cls, response: dict): mandatory_args = { 'key_name': response['keyName'], 'application_key_id': response['applicationKeyId'], 'capabilities': response['capabilities'], 'account_id': response['accountId'], } optional_args = { 'expiration_timestamp_millis': response.get('expirationTimestamp'), 'bucket_id': response.get('bucketId'), 'name_prefix': response.get('namePrefix'), 'options': response.get('options'), } return { **mandatory_args, **{key: value for key, value in optional_args.items() if value is not None}, } def has_capabilities(self, capabilities) -> bool: """ checks whether the key has ALL of the given capabilities """ return len(set(capabilities) - set(self.capabilities)) == 0 def as_dict(self): """Represent the key as a dict, like the one returned by B2 cloud""" mandatory_keys = { 'keyName': self.key_name, 'applicationKeyId': self.id_, 'capabilities': self.capabilities, 'accountId': self.account_id, } optional_keys = { 'expirationTimestamp': self.expiration_timestamp_millis, 'bucketId': self.bucket_id, 'namePrefix': self.name_prefix, 'options': self.options, } return { **mandatory_keys, **{key: value for key, value in optional_keys.items() if value is not None}, } class ApplicationKey(BaseApplicationKey): """Dataclass for storing info about an application key returned by delete-key or list-keys.""" @classmethod def from_api_response(cls, response: dict) -> 'ApplicationKey': """Create an ApplicationKey object from a delete-key or list-key response (a parsed json object).""" return cls(**cls.parse_response_dict(response)) class FullApplicationKey(BaseApplicationKey): """Dataclass for storing info about an application key, including the actual key, as returned by create-key.""" def __init__( self, key_name: str, application_key_id: str, application_key: str, capabilities: List[str], account_id: str, expiration_timestamp_millis: Optional[int] = None, bucket_id: Optional[str] = None, name_prefix: Optional[str] = None, options: Optional[List[str]] = None, ): """ :param key_name: name of the key, assigned by user :param application_key_id: key id, used to authenticate :param application_key: the actual secret key :param capabilities: list of capabilities assigned to this key :param account_id: account's id :param expiration_timestamp_millis: expiration time of the key :param bucket_id: if restricted to a bucket, this is the bucket's id :param name_prefix: if restricted to some files, this is their prefix :param options: reserved for future use """ self.application_key = application_key super().__init__( key_name=key_name, application_key_id=application_key_id, capabilities=capabilities, account_id=account_id, expiration_timestamp_millis=expiration_timestamp_millis, bucket_id=bucket_id, name_prefix=name_prefix, options=options, ) @classmethod def from_create_response(cls, response: dict) -> 'FullApplicationKey': """Create a FullApplicationKey object from a create-key response (a parsed json object).""" return cls(**cls.parse_response_dict(response)) @classmethod def parse_response_dict(cls, response: dict): result = super().parse_response_dict(response) result['application_key'] = response['applicationKey'] return result def as_dict(self): """Represent the key as a dict, like the one returned by B2 cloud""" return { **super().as_dict(), 'applicationKey': self.application_key, } b2-sdk-python-1.17.3/b2sdk/b2http.py000066400000000000000000000460101426424117700167770ustar00rootroot00000000000000###################################################################### # # File: b2sdk/b2http.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from random import random import io import json import logging import socket import arrow import requests from requests.adapters import HTTPAdapter import time from typing import Any, Dict, Optional from .exception import ( B2Error, B2RequestTimeoutDuringUpload, BadDateFormat, BrokenPipe, B2ConnectionError, B2RequestTimeout, ClockSkew, ConnectionReset, interpret_b2_error, UnknownError, UnknownHost ) from .api_config import B2HttpApiConfig, DEFAULT_HTTP_API_CONFIG from .requests import NotDecompressingResponse from .version import USER_AGENT logger = logging.getLogger(__name__) def _print_exception(e, indent=''): """ Used for debugging to print out nested exception structures. :param str indent: message prefix """ print(indent + 'EXCEPTION', repr(e)) print(indent + 'CLASS', type(e)) for (i, a) in enumerate(e.args): print(indent + 'ARG %d: %s' % (i, repr(a))) if isinstance(a, Exception): _print_exception(a, indent + ' ') class ResponseContextManager: """ A context manager that closes a requests.Response when done. """ def __init__(self, response): self.response = response def __enter__(self): return self.response def __exit__(self, exc_type, exc_val, exc_tb): self.response.close() class HttpCallback: """ A callback object that does nothing. Overrides pre_request and/or post_request as desired. """ def pre_request(self, method, url, headers): """ Called before processing an HTTP request. Raises an exception if this request should not be processed. The exception raised must inherit from B2HttpCallbackPreRequestException. :param str method: str, one of: 'POST', 'GET', etc. :param str url: the URL that will be used :param dict headers: the header sent with the request """ def post_request(self, method, url, headers, response): """ Called after processing an HTTP request. Should not raise an exception. Raises an exception if this request should be treated as failing. The exception raised must inherit from B2HttpCallbackPostRequestException. :param str method: one of: 'POST', 'GET', etc. :param str url: the URL that will be used :param dict headers: the header sent with the request :param response: a response object from the requests library """ class ClockSkewHook(HttpCallback): def post_request(self, method, url, headers, response): """ Raise an exception if the clock in the server is too different from the clock on the local host. The Date header contains a string that looks like: "Fri, 16 Dec 2016 20:52:30 GMT". :param str method: one of: 'POST', 'GET', etc. :param str url: the URL that will be used :param dict headers: the header sent with the request :param response: a response object from the requests library """ # Make a string that uses month numbers instead of month names server_date_str = response.headers['Date'] # Convert the server time to a datetime object try: server_time = arrow.get( server_date_str, 'ddd, DD MMM YYYY HH:mm:ss ZZZ' ) # this, unlike datetime.datetime.strptime, always uses English locale except arrow.parser.ParserError: logger.exception('server returned date in an inappropriate format') raise BadDateFormat(server_date_str) # Get the local time local_time = arrow.utcnow() # Check the difference. max_allowed = 10 * 60 # ten minutes, in seconds skew = local_time - server_time skew_seconds = skew.total_seconds() if max_allowed < abs(skew_seconds): raise ClockSkew(skew_seconds) class B2Http: """ A wrapper for the requests module. Provides the operations needed to access B2, and handles retrying when the returned status is 503 Service Unavailable, 429 Too Many Requests, etc. The operations supported are: - post_json_return_json - post_content_return_json - get_content The methods that return JSON either return a Python dict or raise a subclass of B2Error. They can be used like this: .. code-block:: python try: response_dict = b2_http.post_json_return_json(url, headers, params) ... except B2Error as e: ... Please note that the timeout/retry system, including class-level variables, is not a part of the interface and is subject to change. """ TIMEOUT = 128 TIMEOUT_FOR_COPY = 1200 # 20 minutes as server-side copy can take time TIMEOUT_FOR_UPLOAD = 128 TRY_COUNT_DATA = 20 TRY_COUNT_DOWNLOAD = 20 TRY_COUNT_HEAD = 5 TRY_COUNT_OTHER = 5 def __init__(self, api_config: B2HttpApiConfig = DEFAULT_HTTP_API_CONFIG): """ Initialize with a reference to the requests module, which makes it easy to mock for testing. """ self.user_agent = self._get_user_agent(api_config.user_agent_append) self.session = api_config.http_session_factory() if not api_config.decode_content: self.session.adapters.clear() self.session.mount('', NotDecompressingHTTPAdapter()) self.callbacks = [] if api_config.install_clock_skew_hook: self.add_callback(ClockSkewHook()) def add_callback(self, callback): """ Add a callback that inherits from HttpCallback. :param callback: a callback to be added to a chain :type callback: callable """ self.callbacks.append(callback) def post_content_return_json( self, url, headers, data, try_count: int = TRY_COUNT_DATA, post_params=None, _timeout: Optional[int] = None, ): """ Use like this: .. code-block:: python try: response_dict = b2_http.post_content_return_json(url, headers, data) ... except B2Error as e: ... :param str url: a URL to call :param dict headers: headers to send. :param data: bytes (Python 3) or str (Python 2), or a file-like object, to send :return: a dict that is the decoded JSON :rtype: dict """ request_headers = {**headers, 'User-Agent': self.user_agent} # Do the HTTP POST. This may retry, so each post needs to # rewind the data back to the beginning. def do_post(): data.seek(0) self._run_pre_request_hooks('POST', url, request_headers) response = self.session.post( url, headers=request_headers, data=data, timeout=_timeout or self.TIMEOUT_FOR_UPLOAD, ) self._run_post_request_hooks('POST', url, request_headers, response) return response try: response = self._translate_and_retry(do_post, try_count, post_params) except B2RequestTimeout: # this forces a token refresh, which is necessary if request is still alive # on the server but has terminated for some reason on the client. See #79 raise B2RequestTimeoutDuringUpload() # Decode the JSON that came back. If we've gotten this far, # we know we have a status of 200 OK. In this case, the body # of the response is always JSON, so we don't need to handle # it being something else. try: return json.loads(response.content.decode('utf-8')) finally: response.close() def post_json_return_json(self, url, headers, params, try_count: int = TRY_COUNT_OTHER): """ Use like this: .. code-block:: python try: response_dict = b2_http.post_json_return_json(url, headers, params) ... except B2Error as e: ... :param str url: a URL to call :param dict headers: headers to send. :param dict params: a dict that will be converted to JSON :return: the decoded JSON document :rtype: dict """ # This is not just b2_copy_file or b2_copy_part, but it would not # be good to find out by analyzing the url. # In the future a more generic system between raw_api and b2http # to indicate the timeouts should be designed. timeout = self.TIMEOUT_FOR_COPY data = io.BytesIO(json.dumps(params).encode()) return self.post_content_return_json( url, headers, data, try_count, params, _timeout=timeout, ) def get_content(self, url, headers, try_count: int = TRY_COUNT_DOWNLOAD): """ Fetches content from a URL. Use like this: .. code-block:: python try: with b2_http.get_content(url, headers) as response: for byte_data in response.iter_content(chunk_size=1024): ... except B2Error as e: ... The response object is only guarantee to have: - headers - iter_content() :param str url: a URL to call :param dict headers: headers to send :param int try_count: a number or retries :return: Context manager that returns an object that supports iter_content() """ request_headers = {**headers, 'User-Agent': self.user_agent} # Do the HTTP GET. def do_get(): self._run_pre_request_hooks('GET', url, request_headers) response = self.session.get( url, headers=request_headers, stream=True, timeout=self.TIMEOUT ) self._run_post_request_hooks('GET', url, request_headers, response) return response response = self._translate_and_retry(do_get, try_count, None) return ResponseContextManager(response) def head_content( self, url: str, headers: Dict[str, Any], try_count: int = TRY_COUNT_HEAD, ) -> Dict[str, Any]: """ Does a HEAD instead of a GET for the URL. The response's content is limited to the headers. Use like this: .. code-block:: python try: response_dict = b2_http.head_content(url, headers) ... except B2Error as e: ... The response object is only guaranteed to have: - headers :param str url: a URL to call :param dict headers: headers to send :param int try_count: a number or retries :return: the decoded response :rtype: dict """ request_headers = {**headers, 'User-Agent': self.user_agent} # Do the HTTP HEAD. def do_head(): self._run_pre_request_hooks('HEAD', url, request_headers) response = self.session.head( url, headers=request_headers, stream=True, timeout=self.TIMEOUT ) self._run_post_request_hooks('HEAD', url, request_headers, response) return response return self._translate_and_retry(do_head, try_count, None) @classmethod def _get_user_agent(cls, user_agent_append): if user_agent_append: return '%s %s' % (USER_AGENT, user_agent_append) return USER_AGENT def _run_pre_request_hooks(self, method, url, headers): for callback in self.callbacks: callback.pre_request(method, url, headers) def _run_post_request_hooks(self, method, url, headers, response): for callback in self.callbacks: callback.post_request(method, url, headers, response) @classmethod def _translate_errors(cls, fcn, post_params=None): """ Call the given function, turning any exception raised into the right kind of B2Error. :param dict post_params: request parameters """ try: response = fcn() if response.status_code not in [200, 206]: # Decode the error object returned by the service error = json.loads(response.content.decode('utf-8')) if response.content else {} extra_error_keys = error.keys() - ('code', 'status', 'message') if extra_error_keys: logger.debug( 'received error has extra (unsupported) keys: %s', extra_error_keys ) raise interpret_b2_error( int(error.get('status', response.status_code)), error.get('code'), error.get('message'), response.headers, post_params, ) return response except B2Error: raise # pass through exceptions from just above except requests.ConnectionError as e0: e1 = e0.args[0] if isinstance(e1, requests.packages.urllib3.exceptions.MaxRetryError): msg = e1.args[0] if 'nodename nor servname provided, or not known' in msg: # Unknown host, or DNS failing. In the context of calling # B2, this means that something is down between here and # Backblaze, so we treat it like 503 Service Unavailable. raise UnknownHost() elif isinstance(e1, requests.packages.urllib3.exceptions.ProtocolError): e2 = e1.args[1] if isinstance(e2, socket.error): if len(e2.args) >= 2 and e2.args[1] == 'Broken pipe': # Broken pipes are usually caused by the service rejecting # an upload request for cause, so we use a 400 Bad Request # code. raise BrokenPipe() raise B2ConnectionError(str(e0)) except requests.Timeout as e: raise B2RequestTimeout(str(e)) except Exception as e: text = repr(e) # This is a special case to handle when urllib3 doesn't translate # ECONNRESET into something that requests can turn into something # we understand. The SysCallError is from the optional library # pyOpenSsl, which we don't require, so we can't import it and # catch it explicitly. # # The text from one such error looks like this: SysCallError(104, 'ECONNRESET') if text.startswith('SysCallError'): if 'ECONNRESET' in text: raise ConnectionReset() logger.exception('_translate_errors has intercepted an unexpected exception') raise UnknownError(text) @classmethod def _translate_and_retry(cls, fcn, try_count, post_params=None): """ Try calling fcn try_count times, retrying only if the exception is a retryable B2Error. :param int try_count: a number of retries :param dict post_params: request parameters """ # For all but the last try, catch the exception. wait_time = 1.0 max_wait_time = 64 for _ in range(try_count - 1): try: return cls._translate_errors(fcn, post_params) except B2Error as e: if not e.should_retry_http(): raise logger.debug(str(e), exc_info=True) if e.retry_after_seconds is not None: sleep_duration = e.retry_after_seconds sleep_reason = 'server asked us to' else: sleep_duration = wait_time sleep_reason = 'that is what the default exponential backoff is' logger.info( 'Pausing thread for %i seconds because %s', sleep_duration, sleep_reason, ) time.sleep(sleep_duration) # Set up wait time for the next iteration wait_time *= 1.5 if wait_time > max_wait_time: # avoid clients synchronizing and causing a wave # of requests when connectivity is restored wait_time = max_wait_time + random() # If the last try gets an exception, it will be raised. return cls._translate_errors(fcn, post_params) class NotDecompressingHTTPAdapter(HTTPAdapter): """ HTTP adapter that uses :class:`b2sdk.requests.NotDecompressingResponse` instead of the default :code:`requests.Response` class. """ def build_response(self, req, resp): return NotDecompressingResponse.from_builtin_response(super().build_response(req, resp)) def test_http(): """ Run a few tests on error diagnosis. This test takes a while to run and is not used in the automated tests during building. Run the test by hand to exercise the code. """ from .exception import BadJson b2_http = B2Http() # Error from B2 print('TEST: error object from B2') try: b2_http.post_json_return_json( 'https://api.backblazeb2.com/b2api/v1/b2_get_file_info', {}, {} ) assert False, 'should have failed with bad json' except BadJson as e: assert str(e) == 'Bad request: required field fileId is missing' # Successful get print('TEST: get') with b2_http.get_content( 'https://api.backblazeb2.com/test/echo_zeros?length=10', {} ) as response: assert response.status_code == 200 response_data = b''.join(response.iter_content()) assert response_data == b'\x00' * 10 # Successful post print('TEST: post') response_dict = b2_http.post_json_return_json( 'https://api.backblazeb2.com/api/build_version', {}, {} ) assert 'timestamp' in response_dict # Unknown host print('TEST: unknown host') try: b2_http.post_json_return_json('https://unknown.backblazeb2.com', {}, {}) assert False, 'should have failed with unknown host' except UnknownHost: pass # Broken pipe print('TEST: broken pipe') try: data = io.BytesIO(b'\x00' * 10000000) b2_http.post_content_return_json('https://api.backblazeb2.com/bad_url', {}, data) assert False, 'should have failed with broken pipe' except BrokenPipe: pass # Generic connection error print('TEST: generic connection error') try: with b2_http.get_content('https://www.backblazeb2.com:80/bad_url', {}) as response: assert False, 'should have failed with connection error' response.iter_content() # make pyflakes happy except B2ConnectionError: pass b2-sdk-python-1.17.3/b2sdk/bounded_queue_executor.py000066400000000000000000000043371426424117700223440ustar00rootroot00000000000000###################################################################### # # File: b2sdk/bounded_queue_executor.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import threading class BoundedQueueExecutor: """ Wrap a concurrent.futures.Executor and limits the number of requests that can be queued at once. Requests to submit() tasks block until there is room in the queue. The number of available slots in the queue is tracked with a semaphore that is acquired before queueing an action, and released when an action finishes. Counts the number of exceptions thrown by tasks, and makes them available from get_num_exceptions() after shutting down. """ def __init__(self, executor, queue_limit): """ :param executor: an executor to be wrapped :type executor: concurrent.futures.Executor :param queue_limit: a queue limit :type queue_limit: int """ self.executor = executor self.semaphore = threading.Semaphore(queue_limit) self.num_exceptions = 0 def submit(self, fcn, *args, **kwargs): """ Start execution of a callable with the given optional and positional arguments :param fcn: a callable object :type fcn: callable :return: a future object :rtype: concurrent.futures.Future """ # Wait until there is room in the queue. self.semaphore.acquire() # Wrap the action in a function that will release # the semaphore after it runs. def run_it(): try: return fcn(*args, **kwargs) except Exception: self.num_exceptions += 1 raise finally: self.semaphore.release() # Submit the wrapped action. return self.executor.submit(run_it) def shutdown(self): """ Shut an executor down. """ self.executor.shutdown() def get_num_exceptions(self): """ Return a number of exceptions. :rtype: int """ return self.num_exceptions b2-sdk-python-1.17.3/b2sdk/bucket.py000066400000000000000000001357011426424117700170570ustar00rootroot00000000000000###################################################################### # # File: b2sdk/bucket.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging from typing import Optional, Tuple from .encryption.setting import EncryptionSetting, EncryptionSettingFactory from .encryption.types import EncryptionMode from .exception import ( BucketIdNotFound, CopySourceTooBig, FileNotPresent, FileOrBucketNotFound, UnexpectedCloudBehaviour, UnrecognizedBucketType, ) from .file_lock import ( UNKNOWN_BUCKET_RETENTION, BucketRetentionSetting, FileLockConfiguration, FileRetentionSetting, LegalHold, ) from .file_version import DownloadVersion, FileVersion from .progress import AbstractProgressListener, DoNothingProgressListener from .replication.setting import ReplicationConfiguration, ReplicationConfigurationFactory from .transfer.emerge.executor import AUTO_CONTENT_TYPE from .transfer.emerge.write_intent import WriteIntent from .transfer.inbound.downloaded_file import DownloadedFile from .transfer.outbound.copy_source import CopySource from .transfer.outbound.upload_source import UploadSourceBytes, UploadSourceLocalFile from .utils import ( B2TraceMeta, b2_url_encode, disable_trace, limit_trace_arguments, validate_b2_file_name, ) logger = logging.getLogger(__name__) class Bucket(metaclass=B2TraceMeta): """ Provide access to a bucket in B2: listing files, uploading and downloading. """ DEFAULT_CONTENT_TYPE = AUTO_CONTENT_TYPE def __init__( self, api, id_, name=None, type_=None, bucket_info=None, cors_rules=None, lifecycle_rules=None, revision=None, bucket_dict=None, options_set=None, default_server_side_encryption: EncryptionSetting = EncryptionSetting( EncryptionMode.UNKNOWN ), default_retention: BucketRetentionSetting = UNKNOWN_BUCKET_RETENTION, is_file_lock_enabled: Optional[bool] = None, replication: Optional[ReplicationConfiguration] = None, ): """ :param b2sdk.v2.B2Api api: an API object :param str id_: a bucket id :param str name: a bucket name :param str type_: a bucket type :param dict bucket_info: an info to store with a bucket :param dict cors_rules: CORS rules to store with a bucket :param list lifecycle_rules: lifecycle rules of the bucket :param int revision: a bucket revision number :param dict bucket_dict: a dictionary which contains bucket parameters :param set options_set: set of bucket options strings :param b2sdk.v2.EncryptionSetting default_server_side_encryption: default server side encryption settings :param b2sdk.v2.BucketRetentionSetting default_retention: default retention setting :param bool is_file_lock_enabled: whether file locking is enabled or not :param b2sdk.v2.ReplicationConfiguration replication: replication rules for the bucket """ self.api = api self.id_ = id_ self.name = name self.type_ = type_ self.bucket_info = bucket_info or {} self.cors_rules = cors_rules or [] self.lifecycle_rules = lifecycle_rules or [] self.revision = revision self.bucket_dict = bucket_dict or {} self.options_set = options_set or set() self.default_server_side_encryption = default_server_side_encryption self.default_retention = default_retention self.is_file_lock_enabled = is_file_lock_enabled self.replication = replication def get_fresh_state(self) -> 'Bucket': """ Fetch all the information about this bucket and return a new bucket object. This method does NOT change the object it is called on. """ buckets_found = self.api.list_buckets(bucket_id=self.id_) if not buckets_found: raise BucketIdNotFound(self.id_) return buckets_found[0] def get_id(self) -> str: """ Return bucket ID. :rtype: str """ return self.id_ def set_info(self, new_bucket_info, if_revision_is=None) -> 'Bucket': """ Update bucket info. :param dict new_bucket_info: new bucket info dictionary :param int if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is* """ return self.update(bucket_info=new_bucket_info, if_revision_is=if_revision_is) def set_type(self, bucket_type) -> 'Bucket': """ Update bucket type. :param str bucket_type: a bucket type ("allPublic" or "allPrivate") """ return self.update(bucket_type=bucket_type) def update( self, bucket_type: Optional[str] = None, bucket_info: Optional[dict] = None, cors_rules: Optional[dict] = None, lifecycle_rules: Optional[list] = None, if_revision_is: Optional[int] = None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, replication: Optional[ReplicationConfiguration] = None, ) -> 'Bucket': """ Update various bucket parameters. :param bucket_type: a bucket type, e.g. ``allPrivate`` or ``allPublic`` :param bucket_info: an info to store with a bucket :param cors_rules: CORS rules to store with a bucket :param lifecycle_rules: lifecycle rules to store with a bucket :param if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is* :param default_server_side_encryption: default server side encryption settings (``None`` if unknown) :param default_retention: bucket default retention setting :param replication: replication rules for the bucket; """ account_id = self.api.account_info.get_account_id() return self.api.BUCKET_FACTORY_CLASS.from_api_bucket_dict( self.api, self.api.session.update_bucket( account_id, self.id_, bucket_type=bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, replication=replication, ) ) def cancel_large_file(self, file_id): """ Cancel a large file transfer. :param str file_id: a file ID """ return self.api.cancel_large_file(file_id) def download_file_by_id( self, file_id: str, progress_listener: Optional[AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[EncryptionSetting] = None, ) -> DownloadedFile: """ Download a file by ID. .. note:: download_file_by_id actually belongs in :py:class:`b2sdk.v2.B2Api`, not in :py:class:`b2sdk.v2.Bucket`; we just provide a convenient redirect here :param file_id: a file ID :param progress_listener: a progress listener object to use, or ``None`` to not track progress :param range_: two integer values, start and end offsets :param encryption: encryption settings (``None`` if unknown) """ return self.api.download_file_by_id( file_id, progress_listener, range_=range_, encryption=encryption, ) def download_file_by_name( self, file_name: str, progress_listener: Optional[AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[EncryptionSetting] = None, ) -> DownloadedFile: """ Download a file by name. .. seealso:: :ref:`Synchronizer `, a *high-performance* utility that synchronizes a local folder with a Bucket. :param file_name: a file name :param progress_listener: a progress listener object to use, or ``None`` to not track progress :param range_: two integer values, start and end offsets :param encryption: encryption settings (``None`` if unknown) """ url = self.api.session.get_download_url_by_name(self.name, file_name) return self.api.services.download_manager.download_file_from_url( url, progress_listener, range_, encryption=encryption, ) def get_file_info_by_id(self, file_id: str) -> FileVersion: """ Gets a file version's by ID. :param str file_id: the id of the file who's info will be retrieved. :rtype: generator[b2sdk.v2.FileVersion] """ return self.api.get_file_info(file_id) def get_file_info_by_name(self, file_name: str) -> DownloadVersion: """ Gets a file's DownloadVersion by name. :param str file_name: the name of the file who's info will be retrieved. """ try: return self.api.download_version_factory.from_response_headers( self.api.session.get_file_info_by_name(self.name, file_name) ) except FileOrBucketNotFound: raise FileNotPresent(bucket_name=self.name, file_id_or_name=file_name) def get_download_authorization(self, file_name_prefix, valid_duration_in_seconds): """ Return an authorization token that is valid only for downloading files from the given bucket. :param str file_name_prefix: a file name prefix, only files that match it could be downloaded :param int valid_duration_in_seconds: a token is valid only during this amount of seconds """ response = self.api.session.get_download_authorization( self.id_, file_name_prefix, valid_duration_in_seconds ) return response['authorizationToken'] def list_parts(self, file_id, start_part_number=None, batch_size=None): """ Get a list of all parts that have been uploaded for a given file. :param str file_id: a file ID :param int start_part_number: the first part number to return. defaults to the first part. :param int batch_size: the number of parts to fetch at a time from the server """ return self.api.list_parts(file_id, start_part_number, batch_size) def list_file_versions(self, file_name, fetch_count=None): """ Lists all of the versions for a single file. :param str file_name: the name of the file to list. :param int,None fetch_count: how many entries to list per API call or ``None`` to use the default. Acceptable values: 1 - 10000 :rtype: generator[b2sdk.v2.FileVersion] """ if fetch_count is not None and fetch_count <= 0: # fetch_count equal to 0 means "use API default", which we don't want to support here raise ValueError("unsupported fetch_count value") start_file_name = file_name start_file_id = None session = self.api.session while 1: response = session.list_file_versions( self.id_, start_file_name, start_file_id, fetch_count, file_name ) for entry in response['files']: file_version = self.api.file_version_factory.from_api_response(entry) if file_version.file_name != file_name: # All versions for the requested file name have been listed. return yield file_version start_file_name = response['nextFileName'] start_file_id = response['nextFileId'] if start_file_name is None: return def ls( self, folder_to_list: str = '', latest_only: bool = True, recursive: bool = False, fetch_count: Optional[int] = 10000 ): """ Pretend that folders exist and yields the information about the files in a folder. B2 has a flat namespace for the files in a bucket, but there is a convention of using "/" as if there were folders. This method searches through the flat namespace to find the files and "folders" that live within a given folder. When the `recursive` flag is set, lists all of the files in the given folder, and all of its sub-folders. :param folder_to_list: the name of the folder to list; must not start with "/". Empty string means top-level folder :param latest_only: when ``False`` returns info about all versions of a file, when ``True``, just returns info about the most recent versions :param recursive: if ``True``, list folders recursively :param fetch_count: how many entries to return or ``None`` to use the default. Acceptable values: 1 - 10000 :rtype: generator[tuple[b2sdk.v2.FileVersion, str]] :returns: generator of (file_version, folder_name) tuples .. note:: In case of `recursive=True`, folder_name is returned only for first file in the folder. """ # Every file returned must have a name that starts with the # folder name and a "/". prefix = folder_to_list if prefix != '' and not prefix.endswith('/'): prefix += '/' # Loop until all files in the named directory have been listed. # The starting point of the first list_file_names request is the # prefix we're looking for. The prefix ends with '/', which is # now allowed for file names, so no file name will match exactly, # but the first one after that point is the first file in that # "folder". If the first search doesn't produce enough results, # then we keep calling list_file_names until we get all of the # names in this "folder". current_dir = None start_file_name = prefix start_file_id = None session = self.api.session while True: if latest_only: response = session.list_file_names(self.id_, start_file_name, fetch_count, prefix) else: response = session.list_file_versions( self.id_, start_file_name, start_file_id, fetch_count, prefix ) for entry in response['files']: file_version = self.api.file_version_factory.from_api_response(entry) if not file_version.file_name.startswith(prefix): # We're past the files we care about return after_prefix = file_version.file_name[len(prefix):] if '/' not in after_prefix or recursive: # This is not a folder, so we'll print it out and # continue on. yield file_version, None current_dir = None else: # This is a folder. If it's different than the folder # we're already in, then we can print it. This check # is needed, because all of the files in the folder # will be in the list. folder_with_slash = after_prefix.split('/')[0] + '/' if folder_with_slash != current_dir: folder_name = prefix + folder_with_slash yield file_version, folder_name current_dir = folder_with_slash if response['nextFileName'] is None: # The response says there are no more files in the bucket, # so we can stop. return # Now we need to set up the next search. The response from # B2 has the starting point to continue with the next file, # but if we're in the middle of a "folder", we can skip ahead # to the end of the folder. The character after '/' is '0', # so we'll replace the '/' with a '0' and start there. # # When recursive is True, current_dir is always None. if current_dir is None: start_file_name = response.get('nextFileName') start_file_id = response.get('nextFileId') else: start_file_name = max( response['nextFileName'], prefix + current_dir[:-1] + '0', ) def list_unfinished_large_files(self, start_file_id=None, batch_size=None, prefix=None): """ A generator that yields an :py:class:`b2sdk.v2.UnfinishedLargeFile` for each unfinished large file in the bucket, starting at the given file, filtering by prefix. :param str,None start_file_id: a file ID to start from or None to start from the beginning :param int,None batch_size: max file count :param str,None prefix: file name prefix filter :rtype: generator[b2sdk.v2.UnfinishedLargeFile] """ return self.api.services.large_file.list_unfinished_large_files( self.id_, start_file_id=start_file_id, batch_size=batch_size, prefix=prefix, ) @limit_trace_arguments(skip=('data_bytes',)) def upload_bytes( self, data_bytes, file_name, content_type=None, file_infos=None, progress_listener=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): """ Upload bytes in memory to a B2 file. :param bytes data_bytes: a byte array to upload :param str file_name: a file name to upload bytes to :param str,None content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param dict,None file_infos: a file info to store with the file or ``None`` to not store anything :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not track progress :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :rtype: generator[b2sdk.v2.FileVersion] """ upload_source = UploadSourceBytes(data_bytes) return self.upload( upload_source, file_name, content_type=content_type, file_info=file_infos, progress_listener=progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) def upload_local_file( self, local_file, file_name, content_type=None, file_infos=None, sha1_sum=None, min_part_size=None, progress_listener=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): """ Upload a file on local disk to a B2 file. .. seealso:: :ref:`Synchronizer `, a *high-performance* utility that synchronizes a local folder with a :term:`bucket`. :param str local_file: a path to a file on local disk :param str file_name: a file name of the new B2 file :param str,None content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param dict,None file_infos: a file info to store with the file or ``None`` to not store anything :param str,None sha1_sum: file SHA1 hash or ``None`` to compute it automatically :param int min_part_size: a minimum size of a part :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :rtype: b2sdk.v2.FileVersion """ upload_source = UploadSourceLocalFile(local_path=local_file, content_sha1=sha1_sum) return self.upload( upload_source, file_name, content_type=content_type, file_info=file_infos, min_part_size=min_part_size, progress_listener=progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) def upload( self, upload_source, file_name, content_type=None, file_info=None, min_part_size=None, progress_listener=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): """ Upload a file to B2, retrying as needed. The source of the upload is an UploadSource object that can be used to open (and re-open) the file. The result of opening should be a binary file whose read() method returns bytes. The function `opener` should return a file-like object, and it must be possible to call it more than once in case the upload is retried. :param b2sdk.v2.AbstractUploadSource upload_source: an object that opens the source of the upload :param str file_name: the file name of the new B2 file :param str,None content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param dict,None file_info: a file info to store with the file or ``None`` to not store anything :param int,None min_part_size: the smallest part size to use or ``None`` to determine automatically :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :rtype: b2sdk.v2.FileVersion """ return self.create_file( [WriteIntent(upload_source)], file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, # FIXME: Bucket.upload documents wrong logic recommended_upload_part_size=min_part_size, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) def create_file( self, write_intents, file_name, content_type=None, file_info=None, progress_listener=None, recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, min_part_size=None, max_part_size=None, ): """ Creates a new file in this bucket using an iterable (list, tuple etc) of remote or local sources. Source ranges can overlap and remote sources will be prioritized over local sources (when possible). For more information and usage examples please see :ref:`Advanced usage patterns `. :param list[b2sdk.v2.WriteIntent] write_intents: list of write intents (remote or local sources) :param str new_file_name: file name of the new file :param str,None content_type: content_type for the new file, if ``None`` content_type would be automatically determined or it may be copied if it resolves as single part remote source copy :param dict,None file_info: file_info for the new file, if ``None`` it will be set to empty dict or it may be copied if it resolves as single part remote source copy :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param int,None recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, ``None`` for automatic search for this id :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param int min_part_size: lower limit of part size for the transfer planner, in bytes :param int max_part_size: upper limit of part size for the transfer planner, in bytes """ return self._create_file( self.api.services.emerger.emerge, write_intents, file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, continue_large_file_id=continue_large_file_id, recommended_upload_part_size=recommended_upload_part_size, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, min_part_size=min_part_size, max_part_size=max_part_size, ) def create_file_stream( self, write_intents_iterator, file_name, content_type=None, file_info=None, progress_listener=None, recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, min_part_size=None, max_part_size=None, ): """ Creates a new file in this bucket using a stream of multiple remote or local sources. Source ranges can overlap and remote sources will be prioritized over local sources (when possible). For more information and usage examples please see :ref:`Advanced usage patterns `. :param iterator[b2sdk.v2.WriteIntent] write_intents_iterator: iterator of write intents which are sorted ascending by ``destination_offset`` :param str new_file_name: file name of the new file :param str,None content_type: content_type for the new file, if ``None`` content_type would be automatically determined or it may be copied if it resolves as single part remote source copy :param dict,None file_info: file_info for the new file, if ``None`` it will be set to empty dict or it may be copied if it resolves as single part remote source copy :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param int,None recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, if ``None`` in multipart case it would always start a new large file :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param int min_part_size: lower limit of part size for the transfer planner, in bytes :param int max_part_size: upper limit of part size for the transfer planner, in bytes """ return self._create_file( self.api.services.emerger.emerge_stream, write_intents_iterator, file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, continue_large_file_id=continue_large_file_id, recommended_upload_part_size=recommended_upload_part_size, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, min_part_size=min_part_size, max_part_size=max_part_size, ) def _create_file( self, emerger_method, write_intents_iterable, file_name, content_type=None, file_info=None, progress_listener=None, recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, min_part_size=None, max_part_size=None, ): validate_b2_file_name(file_name) progress_listener = progress_listener or DoNothingProgressListener() return emerger_method( self.id_, write_intents_iterable, file_name, content_type, file_info, progress_listener, recommended_upload_part_size=recommended_upload_part_size, continue_large_file_id=continue_large_file_id, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, min_part_size=min_part_size, max_part_size=max_part_size, ) def concatenate( self, outbound_sources, file_name, content_type=None, file_info=None, progress_listener=None, recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, min_part_size=None, max_part_size=None, ): """ Creates a new file in this bucket by concatenating multiple remote or local sources. :param list[b2sdk.v2.OutboundTransferSource] outbound_sources: list of outbound sources (remote or local) :param str new_file_name: file name of the new file :param str,None content_type: content_type for the new file, if ``None`` content_type would be automatically determined from file name or it may be copied if it resolves as single part remote source copy :param dict,None file_info: file_info for the new file, if ``None`` it will be set to empty dict or it may be copied if it resolves as single part remote source copy :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param int,None recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, ``None`` for automatic search for this id :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param int min_part_size: lower limit of part size for the transfer planner, in bytes :param int max_part_size: upper limit of part size for the transfer planner, in bytes """ return self.create_file( WriteIntent.wrap_sources_iterator(outbound_sources), file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, recommended_upload_part_size=recommended_upload_part_size, continue_large_file_id=continue_large_file_id, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, min_part_size=min_part_size, max_part_size=max_part_size, ) def concatenate_stream( self, outbound_sources_iterator, file_name, content_type=None, file_info=None, progress_listener=None, recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): """ Creates a new file in this bucket by concatenating stream of multiple remote or local sources. :param iterator[b2sdk.v2.OutboundTransferSource] outbound_sources_iterator: iterator of outbound sources :param str new_file_name: file name of the new file :param str,None content_type: content_type for the new file, if ``None`` content_type would be automatically determined or it may be copied if it resolves as single part remote source copy :param dict,None file_info: file_info for the new file, if ``None`` it will be set to empty dict or it may be copied if it resolves as single part remote source copy :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param int,None recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, if ``None`` in multipart case it would always start a new large file :param b2sdk.v2.EncryptionSetting encryption: encryption setting (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting """ return self.create_file_stream( WriteIntent.wrap_sources_iterator(outbound_sources_iterator), file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, recommended_upload_part_size=recommended_upload_part_size, continue_large_file_id=continue_large_file_id, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) def get_download_url(self, filename): """ Get file download URL. :param str filename: a file name :rtype: str """ return "%s/file/%s/%s" % ( self.api.account_info.get_download_url(), b2_url_encode(self.name), b2_url_encode(filename), ) def hide_file(self, file_name): """ Hide a file. :param str file_name: a file name :rtype: b2sdk.v2.FileVersion """ response = self.api.session.hide_file(self.id_, file_name) return self.api.file_version_factory.from_api_response(response) def copy( self, file_id, new_file_name, content_type=None, file_info=None, offset=0, length=None, progress_listener=None, destination_encryption: Optional[EncryptionSetting] = None, source_encryption: Optional[EncryptionSetting] = None, source_file_info: Optional[dict] = None, source_content_type: Optional[str] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, min_part_size=None, max_part_size=None, ): """ Creates a new file in this bucket by (server-side) copying from an existing file. :param str file_id: file ID of existing file to copy from :param str new_file_name: file name of the new file :param str,None content_type: content_type for the new file, if ``None`` and ``b2_copy_file`` will be used content_type will be copied from source file - otherwise content_type would be automatically determined :param dict,None file_info: file_info for the new file, if ``None`` will and ``b2_copy_file`` will be used file_info will be copied from source file - otherwise it will be set to empty dict :param int offset: offset of existing file that copy should start from :param int,None length: number of bytes to copy, if ``None`` then ``offset`` have to be ``0`` and it will use ``b2_copy_file`` without ``range`` parameter so it may fail if file is too large. For large files length have to be specified to use ``b2_copy_part`` instead. :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use for multipart copy, or ``None`` to not report progress :param b2sdk.v2.EncryptionSetting destination_encryption: encryption settings for the destination (``None`` if unknown) :param b2sdk.v2.EncryptionSetting source_encryption: encryption settings for the source (``None`` if unknown) :param dict,None source_file_info: source file's file_info dict, useful when copying files with SSE-C :param str,None source_content_type: source file's content type, useful when copying files with SSE-C :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting for the new file. :param bool legal_hold: legal hold setting for the new file. :param int min_part_size: lower limit of part size for the transfer planner, in bytes :param int max_part_size: upper limit of part size for the transfer planner, in bytes """ copy_source = CopySource( file_id, offset=offset, length=length, encryption=source_encryption, source_file_info=source_file_info, source_content_type=source_content_type, ) if not length: # TODO: it feels like this should be checked on lower level - eg. RawApi validate_b2_file_name(new_file_name) try: progress_listener = progress_listener or DoNothingProgressListener() return self.api.services.copy_manager.copy_file( copy_source, new_file_name, content_type=content_type, file_info=file_info, destination_bucket_id=self.id_, progress_listener=progress_listener, destination_encryption=destination_encryption, source_encryption=source_encryption, file_retention=file_retention, legal_hold=legal_hold, ).result() except CopySourceTooBig as e: copy_source.length = e.size progress_listener = DoNothingProgressListener() logger.warning( 'a copy of large object of unknown size is upgraded to the large file interface. No progress report will be provided.' ) return self.create_file( [WriteIntent(copy_source)], new_file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, encryption=destination_encryption, file_retention=file_retention, legal_hold=legal_hold, min_part_size=min_part_size, max_part_size=max_part_size, ) def delete_file_version(self, file_id, file_name): """ Delete a file version. :param str file_id: a file ID :param str file_name: a file name """ # filename argument is not first, because one day it may become optional return self.api.delete_file_version(file_id, file_name) @disable_trace def as_dict(self): """ Return bucket representation as a dictionary. :rtype: dict """ result = { 'accountId': self.api.account_info.get_account_id(), 'bucketId': self.id_, } if self.name is not None: result['bucketName'] = self.name if self.type_ is not None: result['bucketType'] = self.type_ result['bucketInfo'] = self.bucket_info result['corsRules'] = self.cors_rules result['lifecycleRules'] = self.lifecycle_rules result['revision'] = self.revision result['options'] = self.options_set result['defaultServerSideEncryption'] = self.default_server_side_encryption.as_dict() result['isFileLockEnabled'] = self.is_file_lock_enabled result['defaultRetention'] = self.default_retention.as_dict() result['replication'] = self.replication and self.replication.as_dict() return result def __repr__(self): return 'Bucket<%s,%s,%s>' % (self.id_, self.name, self.type_) class BucketFactory: """ This is a factory for creating bucket objects from different kind of objects. """ BUCKET_CLASS = staticmethod(Bucket) @classmethod def from_api_response(cls, api, response): """ Create a Bucket object from API response. :param b2sdk.v2.B2Api api: API object :param requests.Response response: response object :rtype: b2sdk.v2.Bucket """ return [cls.from_api_bucket_dict(api, bucket_dict) for bucket_dict in response['buckets']] @classmethod def from_api_bucket_dict(cls, api, bucket_dict): """ Turn a dictionary, like this: .. code-block:: python { "bucketType": "allPrivate", "bucketId": "a4ba6a39d8b6b5fd561f0010", "bucketName": "zsdfrtsazsdfafr", "accountId": "4aa9865d6f00", "bucketInfo": {}, "options": [], "revision": 1, "defaultServerSideEncryption": { "isClientAuthorizedToRead" : true, "value": { "algorithm" : "AES256", "mode" : "SSE-B2" } }, "fileLockConfiguration": { "isClientAuthorizedToRead": true, "value": { "defaultRetention": { "mode": null, "period": null }, "isFileLockEnabled": false } }, "replicationConfiguration": { "clientIsAllowedToRead": true, "value": { "asReplicationSource": { "replicationRules": [ { "destinationBucketId": "c5f35d53a90a7ea284fb0719", "fileNamePrefix": "", "includeExistingFiles": True, "isEnabled": true, "priority": 1, "replicationRuleName": "replication-us-west" }, { "destinationBucketId": "55f34d53a96a7ea284fb0719", "fileNamePrefix": "", "includeExistingFiles": True, "isEnabled": true, "priority": 2, "replicationRuleName": "replication-us-west-2" } ], "sourceApplicationKeyId": "10053d55ae26b790000000006" }, "asReplicationDestination": { "sourceToDestinationKeyMapping": { "10053d55ae26b790000000045": "10053d55ae26b790000000004", "10053d55ae26b790000000046": "10053d55ae26b790030000004" } } } } } into a Bucket object. :param b2sdk.v2.B2Api api: API client :param dict bucket_dict: a dictionary with bucket properties :rtype: b2sdk.v2.Bucket """ type_ = bucket_dict['bucketType'] if type_ is None: raise UnrecognizedBucketType(bucket_dict['bucketType']) bucket_name = bucket_dict['bucketName'] bucket_id = bucket_dict['bucketId'] bucket_info = bucket_dict['bucketInfo'] cors_rules = bucket_dict['corsRules'] lifecycle_rules = bucket_dict['lifecycleRules'] revision = bucket_dict['revision'] options = set(bucket_dict['options']) if 'defaultServerSideEncryption' not in bucket_dict: raise UnexpectedCloudBehaviour('server did not provide `defaultServerSideEncryption`') default_server_side_encryption = EncryptionSettingFactory.from_bucket_dict(bucket_dict) file_lock_configuration = FileLockConfiguration.from_bucket_dict(bucket_dict) replication = ReplicationConfigurationFactory.from_bucket_dict(bucket_dict).value return cls.BUCKET_CLASS( api, bucket_id, bucket_name, type_, bucket_info, cors_rules, lifecycle_rules, revision, bucket_dict, options, default_server_side_encryption, file_lock_configuration.default_retention, file_lock_configuration.is_file_lock_enabled, replication, ) b2-sdk-python-1.17.3/b2sdk/cache.py000066400000000000000000000057701426424117700166470ustar00rootroot00000000000000###################################################################### # # File: b2sdk/cache.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import ABCMeta, abstractmethod from typing import Optional class AbstractCache(metaclass=ABCMeta): def clear(self): self.set_bucket_name_cache(tuple()) @abstractmethod def get_bucket_id_or_none_from_bucket_name(self, name): pass @abstractmethod def get_bucket_name_or_none_from_allowed(self): pass @abstractmethod def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> Optional[str]: pass @abstractmethod def save_bucket(self, bucket): pass @abstractmethod def set_bucket_name_cache(self, buckets): pass def _name_id_iterator(self, buckets): return ((bucket.name, bucket.id_) for bucket in buckets) class DummyCache(AbstractCache): """ A cache that does nothing. """ def get_bucket_id_or_none_from_bucket_name(self, name): return None def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> Optional[str]: return None def get_bucket_name_or_none_from_allowed(self): return None def save_bucket(self, bucket): pass def set_bucket_name_cache(self, buckets): pass class InMemoryCache(AbstractCache): """ A cache that stores the information in memory. """ def __init__(self): self.name_id_map = {} self.bucket_name = '' def get_bucket_id_or_none_from_bucket_name(self, name): return self.name_id_map.get(name) def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> Optional[str]: for name, cached_id_ in self.name_id_map.items(): if cached_id_ == bucket_id: return name return None def get_bucket_name_or_none_from_allowed(self): return self.bucket_name def save_bucket(self, bucket): self.name_id_map[bucket.name] = bucket.id_ def set_bucket_name_cache(self, buckets): self.name_id_map = dict(self._name_id_iterator(buckets)) class AuthInfoCache(AbstractCache): """ A cache that stores data persistently in StoredAccountInfo. """ def __init__(self, info): self.info = info def get_bucket_id_or_none_from_bucket_name(self, name): return self.info.get_bucket_id_or_none_from_bucket_name(name) def get_bucket_name_or_none_from_bucket_id(self, bucket_id) -> Optional[str]: return self.info.get_bucket_name_or_none_from_bucket_id(bucket_id) def get_bucket_name_or_none_from_allowed(self): return self.info.get_bucket_name_or_none_from_allowed() def save_bucket(self, bucket): self.info.save_bucket(bucket) def set_bucket_name_cache(self, buckets): self.info.refresh_entire_bucket_name_cache(self._name_id_iterator(buckets)) b2-sdk-python-1.17.3/b2sdk/encryption/000077500000000000000000000000001426424117700174135ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/encryption/__init__.py000066400000000000000000000004471426424117700215310ustar00rootroot00000000000000###################################################################### # # File: b2sdk/encryption/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/encryption/setting.py000066400000000000000000000305761426424117700214550ustar00rootroot00000000000000###################################################################### # # File: b2sdk/encryption/setting.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import enum import logging from typing import Optional, Union import urllib import urllib.parse from ..http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME, SSE_C_KEY_ID_HEADER from ..utils import b64_of_bytes, md5_of_bytes from .types import ENCRYPTION_MODES_WITH_MANDATORY_ALGORITHM, ENCRYPTION_MODES_WITH_MANDATORY_KEY from .types import EncryptionAlgorithm, EncryptionMode logger = logging.getLogger(__name__) class _UnknownKeyId(enum.Enum): """The purpose of this enum is to provide a sentinel that can be used with type annotations.""" unknown_key_id = 0 UNKNOWN_KEY_ID = _UnknownKeyId.unknown_key_id """ Value for EncryptionKey.key_id that signifies that the key id may or may not be defined. Useful when establishing encryption settings based on user input (and not based on B2 cloud data). """ class EncryptionKey: """ Hold information about encryption key: the key itself, and its id. The id may be None, if it's not set in encrypted file's fileInfo, or UNKNOWN_KEY_ID when that information is missing. The secret may be None, if encryption metadata is read from the server. """ SECRET_REPR = '******' def __init__(self, secret: Optional[bytes], key_id: Union[str, None, _UnknownKeyId]): self.secret = secret self.key_id = key_id def __eq__(self, other): return self.secret == other.secret and self.key_id == other.key_id def __repr__(self): key_repr = self.SECRET_REPR if self.secret is None: key_repr = None if self.key_id is UNKNOWN_KEY_ID: key_id_repr = 'unknown' else: key_id_repr = repr(self.key_id) return '<%s(%s, %s)>' % (self.__class__.__name__, key_repr, key_id_repr) def as_dict(self): """ Dump EncryptionKey as dict for serializing a to json for requests. """ if self.secret is not None: return { 'customerKey': self.key_b64(), 'customerKeyMd5': self.key_md5(), } return { 'customerKey': self.SECRET_REPR, 'customerKeyMd5': self.SECRET_REPR, } def key_b64(self): return b64_of_bytes(self.secret) def key_md5(self): return b64_of_bytes(md5_of_bytes(self.secret)) class EncryptionSetting: """ Hold information about encryption mode, algorithm and key (for bucket default, file version info or even upload) """ def __init__( self, mode: EncryptionMode, algorithm: EncryptionAlgorithm = None, key: EncryptionKey = None, ): """ :param b2sdk.v2.EncryptionMode mode: encryption mode :param b2sdk.v2.EncryptionAlgorithm algorithm: encryption algorithm :param b2sdk.v2.EncryptionKey key: encryption key object for SSE-C """ self.mode = mode self.algorithm = algorithm self.key = key if self.mode == EncryptionMode.NONE and (self.algorithm or self.key): raise ValueError("cannot specify algorithm or key for 'plaintext' encryption mode") if self.mode in ENCRYPTION_MODES_WITH_MANDATORY_ALGORITHM and not self.algorithm: raise ValueError('must specify algorithm for encryption mode %s' % (self.mode,)) if self.mode in ENCRYPTION_MODES_WITH_MANDATORY_KEY and not self.key: raise ValueError( 'must specify key for encryption mode %s and algorithm %s' % (self.mode, self.algorithm) ) def __eq__(self, other): if other is None: raise ValueError('cannot compare a known encryption setting to an unknown one') return self.mode == other.mode and self.algorithm == other.algorithm and self.key == other.key def serialize_to_json_for_request(self): if self.key and self.key.secret is None: raise ValueError('cannot use an unknown key in requests') return self.as_dict() def as_dict(self): """ Represent the setting as a dict, for example: .. code-block:: python { 'mode': 'SSE-C', 'algorithm': 'AES256', 'customerKey': 'U3hWbVlxM3Q2djl5JEImRSlIQE1jUWZUalduWnI0dTc=', 'customerKeyMd5': 'SWx9GFv5BTT1jdwf48Bx+Q==' } .. code-block:: python { 'mode': 'SSE-B2', 'algorithm': 'AES256' } or .. code-block:: python { 'mode': 'none' } """ result = {'mode': self.mode.value} if self.algorithm is not None: result['algorithm'] = self.algorithm.value if self.mode == EncryptionMode.SSE_C: result.update(self.key.as_dict()) return result def add_to_upload_headers(self, headers): if self.mode == EncryptionMode.NONE: # as of 2021-03-16, server always fails it headers['X-Bz-Server-Side-Encryption'] = self.mode.name elif self.mode == EncryptionMode.SSE_B2: self._add_sse_b2_headers(headers) elif self.mode == EncryptionMode.SSE_C: if self.key.key_id is UNKNOWN_KEY_ID: raise ValueError('Cannot upload a file with an unknown key id') self._add_sse_c_headers(headers) if self.key.key_id is not None: header = SSE_C_KEY_ID_HEADER if headers.get(header) is not None and headers[header] != self.key.key_id: raise ValueError( 'Ambiguous key id set: "%s" in headers and "%s" in %s' % (headers[header], self.key.key_id, self.__class__.__name__) ) headers[header] = urllib.parse.quote(str(self.key.key_id)) else: raise NotImplementedError('unsupported encryption setting: %s' % (self,)) def add_to_download_headers(self, headers): if self.mode == EncryptionMode.NONE: return elif self.mode == EncryptionMode.SSE_B2: self._add_sse_b2_headers(headers) elif self.mode == EncryptionMode.SSE_C: self._add_sse_c_headers(headers) else: raise NotImplementedError('unsupported encryption setting: %s' % (self,)) def _add_sse_b2_headers(self, headers): headers['X-Bz-Server-Side-Encryption'] = self.algorithm.name def _add_sse_c_headers(self, headers): if self.key.secret is None: raise ValueError('Cannot use an unknown key in http headers') headers['X-Bz-Server-Side-Encryption-Customer-Algorithm'] = self.algorithm.name headers['X-Bz-Server-Side-Encryption-Customer-Key'] = self.key.key_b64() headers['X-Bz-Server-Side-Encryption-Customer-Key-Md5'] = self.key.key_md5() def add_key_id_to_file_info(self, file_info: Optional[dict]): if self.key is None or self.key.key_id is None: return file_info if self.key.key_id is UNKNOWN_KEY_ID: raise ValueError('Cannot add an unknown key id to file info') if file_info is None: file_info = {} if file_info.get(SSE_C_KEY_ID_FILE_INFO_KEY_NAME) is not None and file_info[ SSE_C_KEY_ID_FILE_INFO_KEY_NAME] != self.key.key_id: raise ValueError( 'Ambiguous key id set: "%s" in file_info and "%s" in %s' % ( file_info[SSE_C_KEY_ID_FILE_INFO_KEY_NAME], self.key.key_id, self.__class__.__name__ ) ) file_info[SSE_C_KEY_ID_FILE_INFO_KEY_NAME] = self.key.key_id return file_info def __repr__(self): return '<%s(%s, %s, %s)>' % (self.__class__.__name__, self.mode, self.algorithm, self.key) class EncryptionSettingFactory: # 2021-03-17: for the bucket the response of the server is: # if authorized to read: # "mode": "none" # or # "mode": "SSE-B2" # if not authorized to read: # isClientAuthorizedToRead is False and there is no value, so no mode # # BUT file_version (get_file_info, list_file_versions, upload_file etc) # if the file is encrypted, then # "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"}, # or # "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-C"}, # if the file is not encrypted, then "serverSideEncryption" is not present at all @classmethod def from_file_version_dict(cls, file_version_dict: dict) -> EncryptionSetting: """ Returns EncryptionSetting for the given file_version_dict retrieved from the api .. code-block:: python ... "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"}, "fileInfo": {"sse_c_key_id": "key-identifier"} ... """ sse = file_version_dict.get('serverSideEncryption') if sse is None: return EncryptionSetting(EncryptionMode.NONE) key_id = None file_info = file_version_dict.get('fileInfo') if file_info is not None and SSE_C_KEY_ID_FILE_INFO_KEY_NAME in file_info: key_id = urllib.parse.unquote(file_info[SSE_C_KEY_ID_FILE_INFO_KEY_NAME]) return cls._from_value_dict(sse, key_id=key_id) @classmethod def from_bucket_dict(cls, bucket_dict: dict) -> Optional[EncryptionSetting]: """ Returns EncryptionSetting for the given bucket dict retrieved from the api, or None if unauthorized Example inputs: .. code-block:: python ... "defaultServerSideEncryption": { "isClientAuthorizedToRead" : true, "value": { "algorithm" : "AES256", "mode" : "SSE-B2" } } ... unset: .. code-block:: python ... "defaultServerSideEncryption": { "isClientAuthorizedToRead" : true, "value": { "mode" : "none" } } ... unknown: .. code-block:: python ... "defaultServerSideEncryption": { "isClientAuthorizedToRead" : false } ... """ default_sse = bucket_dict.get( 'defaultServerSideEncryption', {'isClientAuthorizedToRead': False}, ) if not default_sse['isClientAuthorizedToRead']: return EncryptionSetting(EncryptionMode.UNKNOWN) assert 'value' in default_sse return cls._from_value_dict(default_sse['value']) @classmethod def _from_value_dict(cls, value_dict, key_id=None): kwargs = {} if value_dict is None: kwargs['mode'] = EncryptionMode.NONE else: mode = EncryptionMode(value_dict['mode'] or 'none') kwargs['mode'] = mode algorithm = value_dict.get('algorithm') if algorithm is not None: kwargs['algorithm'] = EncryptionAlgorithm(algorithm) if mode == EncryptionMode.SSE_C: kwargs['key'] = EncryptionKey(key_id=key_id, secret=None) return EncryptionSetting(**kwargs) @classmethod def from_response_headers(cls, headers): if 'X-Bz-Server-Side-Encryption' in headers: mode = EncryptionMode.SSE_B2 algorithm = EncryptionAlgorithm(headers['X-Bz-Server-Side-Encryption']) return EncryptionSetting(mode, algorithm) if 'X-Bz-Server-Side-Encryption-Customer-Algorithm' in headers: mode = EncryptionMode.SSE_C algorithm = EncryptionAlgorithm( headers['X-Bz-Server-Side-Encryption-Customer-Algorithm'] ) key_id = headers.get(SSE_C_KEY_ID_HEADER) key = EncryptionKey(secret=None, key_id=key_id) return EncryptionSetting(mode, algorithm, key) return EncryptionSetting(EncryptionMode.NONE) SSE_NONE = EncryptionSetting(mode=EncryptionMode.NONE,) """ Commonly used "no encryption" setting """ SSE_B2_AES = EncryptionSetting( mode=EncryptionMode.SSE_B2, algorithm=EncryptionAlgorithm.AES256, ) """ Commonly used SSE-B2 setting """ b2-sdk-python-1.17.3/b2sdk/encryption/types.py000066400000000000000000000025541426424117700211370ustar00rootroot00000000000000###################################################################### # # File: b2sdk/encryption/types.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from enum import Enum, unique @unique class EncryptionAlgorithm(Enum): """Encryption algorithm.""" AES256 = 'AES256' def get_length(self) -> int: if self is EncryptionAlgorithm.AES256: return int(256 / 8) raise NotImplementedError() @unique class EncryptionMode(Enum): """Encryption mode.""" UNKNOWN = None #: unknown encryption mode (sdk doesn't know or used key has no rights to know) NONE = "none" #: no encryption (plaintext) SSE_B2 = 'SSE-B2' #: server-side encryption with key maintained by B2 SSE_C = 'SSE-C' #: server-side encryption with key provided by the client #CLIENT = 'CLIENT' #: client-side encryption def can_be_set_as_bucket_default(self): return self in BUCKET_DEFAULT_ENCRYPTION_MODES ENCRYPTION_MODES_WITH_MANDATORY_ALGORITHM = frozenset( (EncryptionMode.SSE_B2, EncryptionMode.SSE_C) ) # yapf: off ENCRYPTION_MODES_WITH_MANDATORY_KEY = frozenset((EncryptionMode.SSE_C,)) # yapf: off BUCKET_DEFAULT_ENCRYPTION_MODES = frozenset((EncryptionMode.NONE, EncryptionMode.SSE_B2)) b2-sdk-python-1.17.3/b2sdk/exception.py000066400000000000000000000420011426424117700175660ustar00rootroot00000000000000###################################################################### # # File: b2sdk/exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import ABCMeta import logging import re from typing import Any, Dict, Optional from .utils import camelcase_to_underscore, trace_call UPLOAD_TOKEN_USED_CONCURRENTLY_ERROR_MESSAGE_RE = re.compile( r'^more than one upload using auth token (?P[^)]+)$' ) COPY_SOURCE_TOO_BIG_ERROR_MESSAGE_RE = re.compile(r'^Copy source too big: (?P[\d]+)$') logger = logging.getLogger(__name__) class B2Error(Exception, metaclass=ABCMeta): def __init__(self, *args, **kwargs): """ Python 2 does not like it when you pass unicode as the message in an exception. We like to use file names in exception messages. To avoid problems, if the message has any non-ascii characters in it, they are replaced with backslash-uNNNN. https://pythonhosted.org/kitchen/unicode-frustrations.html#frustration-5-exceptions """ # If the exception is caused by a b2 server response, # the server MAY have included instructions to pause the thread before issuing any more requests self.retry_after_seconds = None super(B2Error, self).__init__(*args, **kwargs) @property def prefix(self): """ Nice, auto-generated error message prefix. >>> B2SimpleError().prefix 'Simple error' >>> AlreadyFailed().prefix 'Already failed' """ prefix = self.__class__.__name__ if prefix.startswith('B2'): prefix = prefix[2:] prefix = camelcase_to_underscore(prefix).replace('_', ' ') return prefix[0].upper() + prefix[1:] def should_retry_http(self): """ Return true if this is an error that can cause an HTTP call to be retried. """ return False def should_retry_upload(self): """ Return true if this is an error that should tell the upload code to get a new upload URL and try the upload again. """ return False class InvalidUserInput(B2Error): pass class B2SimpleError(B2Error, metaclass=ABCMeta): """ A B2Error with a message prefix. """ def __str__(self): return '%s: %s' % (self.prefix, super(B2SimpleError, self).__str__()) class NotAllowedByAppKeyError(B2SimpleError, metaclass=ABCMeta): """ Base class for errors caused by restrictions on an application key. """ class TransientErrorMixin(metaclass=ABCMeta): def should_retry_http(self): return True def should_retry_upload(self): return True class AlreadyFailed(B2SimpleError): pass class BadDateFormat(B2SimpleError): prefix = 'Date from server' class BadFileInfo(B2SimpleError): pass class BadJson(B2SimpleError): prefix = 'Bad request' class BadUploadUrl(B2SimpleError): def should_retry_upload(self): return True class BrokenPipe(B2Error): def __str__(self): return 'Broken pipe: unable to send entire request' def should_retry_upload(self): return True class CapabilityNotAllowed(NotAllowedByAppKeyError): pass class ChecksumMismatch(TransientErrorMixin, B2Error): def __init__(self, checksum_type, expected, actual): super(ChecksumMismatch, self).__init__() self.checksum_type = checksum_type self.expected = expected self.actual = actual def __str__(self): return '%s checksum mismatch -- bad data' % (self.checksum_type,) class B2HttpCallbackException(B2SimpleError): pass class B2HttpCallbackPostRequestException(B2HttpCallbackException): pass class B2HttpCallbackPreRequestException(B2HttpCallbackException): pass class BucketNotAllowed(NotAllowedByAppKeyError): pass class ClockSkew(B2HttpCallbackPostRequestException): """ The clock on the server differs from the local clock by too much. """ def __init__(self, clock_skew_seconds): """ :param int clock_skew_seconds: The difference: local_clock - server_clock """ super(ClockSkew, self).__init__() self.clock_skew_seconds = clock_skew_seconds def __str__(self): if self.clock_skew_seconds < 0: return 'ClockSkew: local clock is %d seconds behind server' % ( -self.clock_skew_seconds, ) else: return 'ClockSkew; local clock is %d seconds ahead of server' % ( self.clock_skew_seconds, ) class Conflict(B2SimpleError): pass class ConnectionReset(B2Error): def __str__(self): return 'Connection reset' def should_retry_upload(self): return True class B2ConnectionError(TransientErrorMixin, B2SimpleError): pass class B2RequestTimeout(TransientErrorMixin, B2SimpleError): pass class B2RequestTimeoutDuringUpload(B2RequestTimeout): # if a timeout is hit during upload, it is not guaranteed that the the server has released the upload token lock already, so we'll use a new token def should_retry_http(self): return False class DestFileNewer(B2Error): def __init__(self, dest_path, source_path, dest_prefix, source_prefix): super(DestFileNewer, self).__init__() self.dest_path = dest_path self.source_path = source_path self.dest_prefix = dest_prefix self.source_prefix = source_prefix def __str__(self): return 'source file is older than destination: %s%s with a time of %s cannot be synced to %s%s with a time of %s, unless a valid newer_file_mode is provided' % ( self.source_prefix, self.source_path.relative_path, self.source_path.mod_time, self.dest_prefix, self.dest_path.relative_path, self.dest_path.mod_time, ) def should_retry_http(self): return True class DuplicateBucketName(B2SimpleError): prefix = 'Bucket name is already in use' class ResourceNotFound(B2SimpleError): prefix = 'No such file, bucket, or endpoint' class FileOrBucketNotFound(ResourceNotFound): def __init__(self, bucket_name=None, file_id_or_name=None): super(FileOrBucketNotFound, self).__init__() self.bucket_name = bucket_name self.file_id_or_name = file_id_or_name def __str__(self): file_str = ('file [%s]' % self.file_id_or_name) if self.file_id_or_name else 'a file' bucket_str = ('bucket [%s]' % self.bucket_name) if self.bucket_name else 'a bucket' return 'Could not find %s within %s' % (file_str, bucket_str) class BucketIdNotFound(ResourceNotFound): def __init__(self, bucket_id): self.bucket_id = bucket_id def __str__(self): return 'Bucket with id=%s not found' % (self.bucket_id,) class FileAlreadyHidden(B2SimpleError): pass class FileNameNotAllowed(NotAllowedByAppKeyError): pass class FileNotPresent(FileOrBucketNotFound): def __str__(self): # overridden to retain message across prev versions return "File not present%s" % (': ' + self.file_id_or_name if self.file_id_or_name else "") class UnusableFileName(B2SimpleError): """ Raise when a filename doesn't meet the rules. Could possibly use InvalidUploadSource, but this is intended for the filename on the server, which could differ. https://www.backblaze.com/b2/docs/files.html. """ pass class InvalidMetadataDirective(B2Error): pass class SSECKeyIdMismatchInCopy(InvalidMetadataDirective): pass class InvalidRange(B2Error): def __init__(self, content_length, range_): super(InvalidRange, self).__init__() self.content_length = content_length self.range_ = range_ def __str__(self): return 'A range of %d-%d was requested (size of %d), but cloud could only serve %d of that' % ( self.range_[0], self.range_[1], self.range_[1] - self.range_[0] + 1, self.content_length, ) class InvalidUploadSource(B2SimpleError): pass class BadRequest(B2Error): def __init__(self, message, code): super(BadRequest, self).__init__() self.message = message self.code = code def __str__(self): return '%s (%s)' % (self.message, self.code) class CopySourceTooBig(BadRequest): def __init__(self, message, code, size: int): super().__init__(message, code) self.size = size class Unauthorized(B2Error): def __init__(self, message, code): super(Unauthorized, self).__init__() self.message = message self.code = code def __str__(self): return '%s (%s)' % (self.message, self.code) def should_retry_upload(self): return True class EmailNotVerified(Unauthorized): def should_retry_upload(self): return False class InvalidAuthToken(Unauthorized): """ Specific type of Unauthorized that means the auth token is invalid. This is not the case where the auth token is valid, but does not allow access. """ def __init__(self, message, code): super(InvalidAuthToken, self).__init__('Invalid authorization token. Server said: ' + message, code) class RestrictedBucket(B2Error): def __init__(self, bucket_name): super(RestrictedBucket, self).__init__() self.bucket_name = bucket_name def __str__(self): return 'Application key is restricted to bucket: %s' % self.bucket_name class MaxFileSizeExceeded(B2Error): def __init__(self, size, max_allowed_size): super(MaxFileSizeExceeded, self).__init__() self.size = size self.max_allowed_size = max_allowed_size def __str__(self): return 'Allowed file size of exceeded: %s > %s' % ( self.size, self.max_allowed_size, ) class MaxRetriesExceeded(B2Error): def __init__(self, limit, exception_info_list): super(MaxRetriesExceeded, self).__init__() self.limit = limit self.exception_info_list = exception_info_list def __str__(self): exceptions = '\n'.join(str(wrapped_error) for wrapped_error in self.exception_info_list) return 'FAILED to upload after %s tries. Encountered exceptions: %s' % ( self.limit, exceptions, ) class MissingPart(B2SimpleError): prefix = 'Part number has not been uploaded' class NonExistentBucket(FileOrBucketNotFound): def __str__(self): # overridden to retain message across prev versions return "No such bucket%s" % (': ' + self.bucket_name if self.bucket_name else "") class FileSha1Mismatch(B2SimpleError): prefix = 'Upload file SHA1 mismatch' class PartSha1Mismatch(B2Error): def __init__(self, key): super(PartSha1Mismatch, self).__init__() self.key = key def __str__(self): return 'Part number %s has wrong SHA1' % (self.key,) class ServiceError(TransientErrorMixin, B2Error): """ Used for HTTP status codes 500 through 599. """ class CapExceeded(B2Error): def __str__(self): return 'Cap exceeded.' class StorageCapExceeded(CapExceeded): def __str__(self): return 'Cannot upload or copy files, storage cap exceeded.' class TransactionCapExceeded(CapExceeded): def __str__(self): return 'Cannot perform the operation, transaction cap exceeded.' class TooManyRequests(B2Error): def __init__(self, retry_after_seconds=None): super(TooManyRequests, self).__init__() self.retry_after_seconds = retry_after_seconds def __str__(self): return 'Too many requests' def should_retry_http(self): return True class TruncatedOutput(TransientErrorMixin, B2Error): def __init__(self, bytes_read, file_size): super(TruncatedOutput, self).__init__() self.bytes_read = bytes_read self.file_size = file_size def __str__(self): return 'only %d of %d bytes read' % ( self.bytes_read, self.file_size, ) class UnexpectedCloudBehaviour(B2SimpleError): pass class UnknownError(B2SimpleError): pass class UnknownHost(B2Error): def __str__(self): return 'unknown host' class UnrecognizedBucketType(B2Error): pass class UnsatisfiableRange(B2Error): def __str__(self): return "The range in the request is outside the size of the file" class UploadTokenUsedConcurrently(B2Error): def __init__(self, token): super(UploadTokenUsedConcurrently, self).__init__() self.token = token def __str__(self): return "More than one concurrent upload using auth token %s" % (self.token,) class AccessDenied(B2Error): def __str__(self): return "This call with these parameters is not allowed for this auth token" class SSECKeyError(AccessDenied): def __str__(self): return "Wrong or no SSE-C key provided when reading a file." class RetentionWriteError(AccessDenied): def __str__(self): return "Auth token not authorized to write retention or file already in 'compliance' mode or " \ "bypassGovernance=true parameter missing" class WrongEncryptionModeForBucketDefault(InvalidUserInput): def __init__(self, encryption_mode): super().__init__() self.encryption_mode = encryption_mode def __str__(self): return "%s cannot be used as default for a bucket." % (self.encryption_mode,) class CopyArgumentsMismatch(InvalidUserInput): pass @trace_call(logger) def interpret_b2_error( status: int, code: Optional[str], message: Optional[str], response_headers: Dict[str, Any], post_params: Optional[Dict[str, Any]] = None ) -> B2Error: post_params = post_params or {} if status == 400 and code == "already_hidden": return FileAlreadyHidden(post_params.get('fileName')) elif status == 400 and code == 'bad_json': return BadJson(message) elif ( (status == 400 and code in ("no_such_file", "file_not_present")) or (status == 404 and code == "not_found") ): # hide_file returns 400 and "no_such_file" # delete_file_version returns 400 and "file_not_present" # get_file_info returns 404 and "not_found" # download_file_by_name/download_file_by_id return 404 and "not_found" # but don't have post_params return FileNotPresent( file_id_or_name=post_params.get('fileId') or post_params.get('fileName') ) elif status == 404: # often times backblaze will return cryptic error messages on invalid URLs. # We should ideally only reach that case on programming error or outdated # sdk versions, but to prevent user confusion we omit the message param return ResourceNotFound() elif status == 400 and code == "duplicate_bucket_name": return DuplicateBucketName(post_params.get('bucketName')) elif status == 400 and code == "missing_part": return MissingPart(post_params.get('fileId')) elif status == 400 and code == "part_sha1_mismatch": return PartSha1Mismatch(post_params.get('fileId')) elif status == 400 and code == "bad_bucket_id": return BucketIdNotFound(post_params.get('bucketId')) elif status == 400 and code in ('bad_request', 'auth_token_limit', 'source_too_large'): # it's "bad_request" on 2022-03-29, but will become 'auth_token_limit' in 2022-04 # TODO: cleanup after 2022-05-01 matcher = UPLOAD_TOKEN_USED_CONCURRENTLY_ERROR_MESSAGE_RE.match(message) if matcher is not None: token = matcher.group('token') return UploadTokenUsedConcurrently(token) # it's "bad_request" on 2022-03-29, but will become 'source_too_large' in 2022-04 # TODO: cleanup after 2022-05-01 matcher = COPY_SOURCE_TOO_BIG_ERROR_MESSAGE_RE.match(message) if matcher is not None: size = int(matcher.group('size')) return CopySourceTooBig(size) return BadRequest(message, code) elif status == 400: return BadRequest(message, code) elif status == 401 and code in ("bad_auth_token", "expired_auth_token"): return InvalidAuthToken(message, code) elif status == 401 and code == 'email_not_verified': return EmailNotVerified(message, code) elif status == 401: return Unauthorized(message, code) elif status == 403 and code == "storage_cap_exceeded": return StorageCapExceeded() elif status == 403 and code == "transaction_cap_exceeded": return TransactionCapExceeded() elif status == 403 and code == "access_denied": return AccessDenied() elif status == 409: return Conflict() elif status == 416 and code == "range_not_satisfiable": return UnsatisfiableRange() elif status == 429: return TooManyRequests(retry_after_seconds=response_headers.get('retry-after')) elif 500 <= status < 600: return ServiceError('%d %s %s' % (status, code, message)) return UnknownError('%d %s %s' % (status, code, message)) b2-sdk-python-1.17.3/b2sdk/file_lock.py000066400000000000000000000330121426424117700175210ustar00rootroot00000000000000###################################################################### # # File: b2sdk/file_lock.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional import enum from .exception import UnexpectedCloudBehaviour ACTIONS_WITHOUT_LOCK_SETTINGS = frozenset(['hide', 'folder']) @enum.unique class RetentionMode(enum.Enum): """Enum class representing retention modes set in files and buckets""" GOVERNANCE = "governance" #: retention settings for files in this mode can be modified by clients with appropriate application key capabilities COMPLIANCE = "compliance" #: retention settings for files in this mode can only be modified by extending the retention dates by clients with appropriate application key capabilities NONE = None #: retention not set UNKNOWN = "unknown" #: the client is not authorized to read retention settings RETENTION_MODES_REQUIRING_PERIODS = frozenset({RetentionMode.COMPLIANCE, RetentionMode.GOVERNANCE}) class RetentionPeriod: """Represent a time period (either in days or in years) that is used as a default for bucket retention""" KNOWN_UNITS = ['days', 'years'] def __init__(self, years: Optional[int] = None, days: Optional[int] = None): """Create a retention period, provide exactly one of: days, years""" assert (years is None) != (days is None) if years is not None: self.duration = years self.unit = 'years' else: self.duration = days self.unit = 'days' @classmethod def from_period_dict(cls, period_dict): """ Build a RetentionPeriod from an object returned by the server, such as: .. code-block :: { "duration": 2, "unit": "years" } """ assert period_dict['unit'] in cls.KNOWN_UNITS return cls(**{period_dict['unit']: period_dict['duration']}) def as_dict(self): return { "duration": self.duration, "unit": self.unit, } def __repr__(self): return '%s(%s %s)' % (self.__class__.__name__, self.duration, self.unit) def __eq__(self, other): return self.unit == other.unit and self.duration == other.duration class FileRetentionSetting: """Represent file retention settings, i.e. whether the file is retained, in which mode and until when""" def __init__(self, mode: RetentionMode, retain_until: Optional[int] = None): if mode in RETENTION_MODES_REQUIRING_PERIODS and retain_until is None: raise ValueError('must specify retain_until for retention mode %s' % (mode,)) self.mode = mode self.retain_until = retain_until @classmethod def from_file_version_dict(cls, file_version_dict: dict) -> 'FileRetentionSetting': """ Returns FileRetentionSetting for the given file_version_dict retrieved from the api. E.g. .. code-block :: { "action": "upload", "fileRetention": { "isClientAuthorizedToRead": false, "value": null }, ... } { "action": "upload", "fileRetention": { "isClientAuthorizedToRead": true, "value": { "mode": "governance", "retainUntilTimestamp": 1628942493000 } }, ... } """ if 'fileRetention' not in file_version_dict: if file_version_dict['action'] not in ACTIONS_WITHOUT_LOCK_SETTINGS: raise UnexpectedCloudBehaviour( 'No fileRetention provided for file version with action=%s' % (file_version_dict['action']) ) return NO_RETENTION_FILE_SETTING file_retention_dict = file_version_dict['fileRetention'] if not file_retention_dict['isClientAuthorizedToRead']: return cls(RetentionMode.UNKNOWN, None) return cls.from_file_retention_value_dict(file_retention_dict['value']) @classmethod def from_file_retention_value_dict( cls, file_retention_value_dict: dict ) -> 'FileRetentionSetting': mode = file_retention_value_dict['mode'] if mode is None: return NO_RETENTION_FILE_SETTING return cls( RetentionMode(mode), file_retention_value_dict['retainUntilTimestamp'], ) @classmethod def from_server_response(cls, server_response: dict) -> 'FileRetentionSetting': return cls.from_file_retention_value_dict(server_response['fileRetention']) @classmethod def from_response_headers(cls, headers) -> 'FileRetentionSetting': retention_mode_header = 'X-Bz-File-Retention-Mode' retain_until_header = 'X-Bz-File-Retention-Retain-Until-Timestamp' if retention_mode_header in headers: if retain_until_header in headers: retain_until = int(headers[retain_until_header]) else: retain_until = None return cls(RetentionMode(headers[retention_mode_header]), retain_until) if 'X-Bz-Client-Unauthorized-To-Read' in headers and retention_mode_header in headers[ 'X-Bz-Client-Unauthorized-To-Read'].split(','): return UNKNOWN_FILE_RETENTION_SETTING return NO_RETENTION_FILE_SETTING # the bucket is not file-lock-enabled or the file is has no retention set def serialize_to_json_for_request(self): if self.mode is RetentionMode.UNKNOWN: raise ValueError('cannot use an unknown file retention setting in requests') return self.as_dict() def as_dict(self): return { "mode": self.mode.value, "retainUntilTimestamp": self.retain_until, } def add_to_to_upload_headers(self, headers): if self.mode is RetentionMode.UNKNOWN: raise ValueError('cannot use an unknown file retention setting in requests') headers['X-Bz-File-Retention-Mode'] = str( self.mode.value ) # mode = NONE is not supported by the server at the # moment, but it should be headers['X-Bz-File-Retention-Retain-Until-Timestamp'] = str(self.retain_until) def __eq__(self, other): return self.mode == other.mode and self.retain_until == other.retain_until def __repr__(self): return '%s(%s, %s)' % ( self.__class__.__name__, repr(self.mode.value), repr(self.retain_until) ) @enum.unique class LegalHold(enum.Enum): """Enum holding information about legalHold switch in a file.""" ON = 'on' #: legal hold set to "on" OFF = 'off' #: legal hold set to "off" UNSET = None #: server default, as for now it is functionally equivalent to OFF UNKNOWN = 'unknown' #: the client is not authorized to read legal hold settings def is_on(self): """Is the legalHold switch on?""" return self is LegalHold.ON def is_off(self): """Is the legalHold switch off or left as default (which also means off)?""" return self is LegalHold.OFF or self is LegalHold.UNSET def is_unknown(self): """Is the legalHold switch unknown?""" return self is LegalHold.UNKNOWN @classmethod def from_file_version_dict(cls, file_version_dict: dict) -> 'LegalHold': if 'legalHold' not in file_version_dict: if file_version_dict['action'] not in ACTIONS_WITHOUT_LOCK_SETTINGS: raise UnexpectedCloudBehaviour( 'legalHold not provided for file version with action=%s' % (file_version_dict['action']) ) return cls.UNSET if not file_version_dict['legalHold']['isClientAuthorizedToRead']: return cls.UNKNOWN return cls.from_string_or_none(file_version_dict['legalHold']['value']) @classmethod def from_server_response(cls, server_response: dict) -> 'LegalHold': return cls.from_string_or_none(server_response['legalHold']) @classmethod def from_string_or_none(cls, string: Optional[str]) -> 'LegalHold': return cls(string) @classmethod def from_response_headers(cls, headers) -> 'LegalHold': legal_hold_header = 'X-Bz-File-Legal-Hold' if legal_hold_header in headers: return cls(headers['X-Bz-File-Legal-Hold']) if 'X-Bz-Client-Unauthorized-To-Read' in headers and legal_hold_header in headers[ 'X-Bz-Client-Unauthorized-To-Read'].split(','): return cls.UNKNOWN return cls.UNSET # the bucket is not file-lock-enabled or the header is missing for any other reason def to_server(self) -> str: if self.is_unknown(): raise ValueError('Cannot use an unknown legal hold in requests') if self.is_on(): return self.__class__.ON.value return self.__class__.OFF.value def add_to_upload_headers(self, headers): headers['X-Bz-File-Legal-Hold'] = self.to_server() class BucketRetentionSetting: """Represent bucket's default file retention settings, i.e. whether the files should be retained, in which mode and for how long""" def __init__(self, mode: RetentionMode, period: Optional[RetentionPeriod] = None): if mode in RETENTION_MODES_REQUIRING_PERIODS and period is None: raise ValueError('must specify period for retention mode %s' % (mode,)) self.mode = mode self.period = period @classmethod def from_bucket_retention_dict(cls, retention_dict: dict): """ Build a BucketRetentionSetting from an object returned by the server, such as: .. code-block:: { "mode": "compliance", "period": { "duration": 7, "unit": "days" } } """ period = retention_dict['period'] if period is not None: period = RetentionPeriod.from_period_dict(period) return cls(RetentionMode(retention_dict['mode']), period) def as_dict(self): result = { 'mode': self.mode.value, } if self.period is not None: result['period'] = self.period.as_dict() return result def serialize_to_json_for_request(self): if self.mode == RetentionMode.UNKNOWN: raise ValueError('cannot use an unknown file lock configuration in requests') return self.as_dict() def __eq__(self, other): return self.mode == other.mode and self.period == other.period def __repr__(self): return '%s(%s, %s)' % (self.__class__.__name__, repr(self.mode.value), repr(self.period)) class FileLockConfiguration: """Represent bucket's file lock configuration, i.e. whether the file lock mechanism is enabled and default file retention""" def __init__( self, default_retention: BucketRetentionSetting, is_file_lock_enabled: Optional[bool], ): self.default_retention = default_retention self.is_file_lock_enabled = is_file_lock_enabled @classmethod def from_bucket_dict(cls, bucket_dict): """ Build a FileLockConfiguration from an object returned by server, such as: .. code-block:: { "isClientAuthorizedToRead": true, "value": { "defaultRetention": { "mode": "governance", "period": { "duration": 2, "unit": "years" } }, "isFileLockEnabled": true } } or { "isClientAuthorizedToRead": false, "value": null } """ if not bucket_dict['fileLockConfiguration']['isClientAuthorizedToRead']: return cls(UNKNOWN_BUCKET_RETENTION, None) retention = BucketRetentionSetting.from_bucket_retention_dict( bucket_dict['fileLockConfiguration']['value']['defaultRetention'] ) is_file_lock_enabled = bucket_dict['fileLockConfiguration']['value']['isFileLockEnabled'] return cls(retention, is_file_lock_enabled) def as_dict(self): return { "defaultRetention": self.default_retention.as_dict(), "isFileLockEnabled": self.is_file_lock_enabled, } def __eq__(self, other): return self.default_retention == other.default_retention and self.is_file_lock_enabled == other.is_file_lock_enabled def __repr__(self): return '%s(%s, %s)' % ( self.__class__.__name__, repr(self.default_retention), repr(self.is_file_lock_enabled) ) UNKNOWN_BUCKET_RETENTION = BucketRetentionSetting(RetentionMode.UNKNOWN) """Commonly used "unknown" default bucket retention setting""" UNKNOWN_FILE_LOCK_CONFIGURATION = FileLockConfiguration(UNKNOWN_BUCKET_RETENTION, None) """Commonly used "unknown" bucket file lock setting""" NO_RETENTION_BUCKET_SETTING = BucketRetentionSetting(RetentionMode.NONE) """Commonly used "no retention" default bucket retention""" NO_RETENTION_FILE_SETTING = FileRetentionSetting(RetentionMode.NONE) """Commonly used "no retention" file setting""" UNKNOWN_FILE_RETENTION_SETTING = FileRetentionSetting(RetentionMode.UNKNOWN) """Commonly used "unknown" file retention setting""" b2-sdk-python-1.17.3/b2sdk/file_version.py000066400000000000000000000517141426424117700202670ustar00rootroot00000000000000###################################################################### # # File: b2sdk/file_version.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Dict, Optional, Union, Tuple, TYPE_CHECKING import re from copy import deepcopy from .encryption.setting import EncryptionSetting, EncryptionSettingFactory from .replication.types import ReplicationStatus from .http_constants import FILE_INFO_HEADER_PREFIX_LOWER, SRC_LAST_MODIFIED_MILLIS from .file_lock import FileRetentionSetting, LegalHold, NO_RETENTION_FILE_SETTING from .progress import AbstractProgressListener from .utils.range_ import Range from .utils import b2_url_decode if TYPE_CHECKING: from .api import B2Api from .transfer.inbound.downloaded_file import DownloadedFile UNVERIFIED_CHECKSUM_PREFIX = 'unverified:' class BaseFileVersion: """ Base class for representing file metadata in B2 cloud. :ivar size - size of the whole file (for "upload" markers) """ __slots__ = [ 'id_', 'api', 'file_name', 'size', 'content_type', 'content_sha1', 'content_sha1_verified', 'file_info', 'upload_timestamp', 'server_side_encryption', 'legal_hold', 'file_retention', 'mod_time_millis', 'replication_status', ] _TYPE_MATCHER = re.compile('[a-z0-9]+_[a-z0-9]+_f([0-9]).*') _FILE_TYPE = { 1: 'small', 2: 'large', 3: 'part', 4: 'tiny', } def __init__( self, api: 'B2Api', id_: str, file_name: str, size: int, content_type: Optional[str], content_sha1: Optional[str], file_info: Dict[str, str], upload_timestamp: int, server_side_encryption: EncryptionSetting, file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, replication_status: Optional[ReplicationStatus] = None, ): self.api = api self.id_ = id_ self.file_name = file_name self.size = size self.content_type = content_type self.content_sha1, self.content_sha1_verified = self._decode_content_sha1(content_sha1) self.file_info = file_info or {} self.upload_timestamp = upload_timestamp self.server_side_encryption = server_side_encryption self.file_retention = file_retention self.legal_hold = legal_hold self.replication_status = replication_status if SRC_LAST_MODIFIED_MILLIS in self.file_info: self.mod_time_millis = int(self.file_info[SRC_LAST_MODIFIED_MILLIS]) else: self.mod_time_millis = self.upload_timestamp @classmethod def _decode_content_sha1(cls, content_sha1): if content_sha1.startswith(UNVERIFIED_CHECKSUM_PREFIX): return content_sha1[len(UNVERIFIED_CHECKSUM_PREFIX):], False return content_sha1, True @classmethod def _encode_content_sha1(cls, content_sha1, content_sha1_verified): if not content_sha1_verified: return '%s%s' % (UNVERIFIED_CHECKSUM_PREFIX, content_sha1) return content_sha1 def _clone(self, **new_attributes: Dict[str, object]): """ Create new instance based on the old one, overriding attributes with :code:`new_attributes` (only applies to arguments passed to __init__) """ args = self._get_args_for_clone() return self.__class__(**{**args, **new_attributes}) def _get_args_for_clone(self): return { 'api': self.api, 'id_': self.id_, 'file_name': self.file_name, 'size': self.size, 'content_type': self.content_type, 'content_sha1': self._encode_content_sha1(self.content_sha1, self.content_sha1_verified), 'file_info': self.file_info, 'upload_timestamp': self.upload_timestamp, 'server_side_encryption': self.server_side_encryption, 'file_retention': self.file_retention, 'legal_hold': self.legal_hold, 'replication_status': self.replication_status, } # yapf: disable def as_dict(self): """ represents the object as a dict which looks almost exactly like the raw api output for upload/list """ result = { 'fileId': self.id_, 'fileName': self.file_name, 'fileInfo': self.file_info, 'serverSideEncryption': self.server_side_encryption.as_dict(), 'legalHold': self.legal_hold.value, 'fileRetention': self.file_retention.as_dict(), } if self.size is not None: result['size'] = self.size if self.upload_timestamp is not None: result['uploadTimestamp'] = self.upload_timestamp if self.content_type is not None: result['contentType'] = self.content_type if self.content_sha1 is not None: result['contentSha1'] = self._encode_content_sha1( self.content_sha1, self.content_sha1_verified ) result['replicationStatus'] = self.replication_status and self.replication_status.value return result def __eq__(self, other): sentry = object() for attr in self._all_slots(): if getattr(self, attr) != getattr(other, attr, sentry): return False return True def __repr__(self): return '%s(%s)' % ( self.__class__.__name__, ', '.join(repr(getattr(self, attr)) for attr in self._all_slots()) ) def _all_slots(self): """Return all slots for an object (for it's class and all parent classes). Useful in auxiliary methods.""" all_slots = [] for klass in self.__class__.__mro__[-1::-1]: all_slots.extend(getattr(klass, '__slots__', [])) return all_slots def delete(self) -> 'FileIdAndName': return self.api.delete_file_version(self.id_, self.file_name) def update_legal_hold(self, legal_hold: LegalHold) -> 'BaseFileVersion': legal_hold = self.api.update_file_legal_hold(self.id_, self.file_name, legal_hold) return self._clone(legal_hold=legal_hold) def update_retention( self, file_retention: FileRetentionSetting, bypass_governance: bool = False, ) -> 'BaseFileVersion': file_retention = self.api.update_file_retention( self.id_, self.file_name, file_retention, bypass_governance ) return self._clone(file_retention=file_retention) def _type(self): """ FOR TEST PURPOSES ONLY not guaranteed to work for perpetuity (using undocumented server behavior) """ m = self._TYPE_MATCHER.match(self.id_) assert m, self.id_ return self._FILE_TYPE[int(m.group(1))] class FileVersion(BaseFileVersion): """ A structure which represents a version of a file (in B2 cloud). :ivar str ~.id\_: ``fileId`` :ivar str ~.file_name: full file name (with path) :ivar ~.size: size in bytes, can be ``None`` (unknown) :ivar str ~.content_type: RFC 822 content type, for example ``"application/octet-stream"`` :ivar ~.upload_timestamp: in milliseconds since :abbr:`epoch (1970-01-01 00:00:00)`. Can be ``None`` (unknown). :ivar str ~.action: ``"upload"``, ``"hide"`` or ``"delete"`` """ __slots__ = [ 'account_id', 'bucket_id', 'content_md5', 'action', ] # defined at https://www.backblaze.com/b2/docs/files.html#httpHeaderSizeLimit DEFAULT_HEADERS_LIMIT = 7000 ADVANCED_HEADERS_LIMIT = 2048 def __init__( self, api: 'B2Api', id_: str, file_name: str, size: Union[int, None, str], content_type: Optional[str], content_sha1: Optional[str], file_info: Dict[str, str], upload_timestamp: int, account_id: str, bucket_id: str, action: str, content_md5: Optional[str], server_side_encryption: EncryptionSetting, file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, replication_status: Optional[ReplicationStatus] = None, ): self.account_id = account_id self.bucket_id = bucket_id self.content_md5 = content_md5 self.action = action super().__init__( api=api, id_=id_, file_name=file_name, size=size, content_type=content_type, content_sha1=content_sha1, file_info=file_info, upload_timestamp=upload_timestamp, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, replication_status=replication_status, ) def _get_args_for_clone(self): args = super()._get_args_for_clone() args.update( { 'account_id': self.account_id, 'bucket_id': self.bucket_id, 'action': self.action, 'content_md5': self.content_md5, } ) return args def as_dict(self): result = super().as_dict() result['accountId'] = self.account_id result['bucketId'] = self.bucket_id if self.action is not None: result['action'] = self.action if self.content_md5 is not None: result['contentMd5'] = self.content_md5 return result def get_fresh_state(self) -> 'FileVersion': """ Fetch all the information about this file version and return a new FileVersion object. This method does NOT change the object it is called on. """ return self.api.get_file_info(self.id_) def download( self, progress_listener: Optional[AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[EncryptionSetting] = None, ) -> 'DownloadedFile': return self.api.download_file_by_id( self.id_, progress_listener=progress_listener, range_=range_, encryption=encryption, ) def _get_upload_headers(self) -> bytes: """ Return encoded http headers, as when sending an upload request to b2 http api. WARNING: the headers do not contain newlines between headers and spaces between key and value. This implementation is in par with ADVANCED_HEADERS_LIMIT and is reasonable only for `has_large_header` method """ # sometimes secret is not available, but we want to calculate headers # size anyway; to bypass this, we use a fake encryption setting # with a fake key sse = self.server_side_encryption if sse and sse.key and sse.key.secret is None: sse = deepcopy(sse) sse.key.secret = b'*' * sse.algorithm.get_length() headers = self.api.raw_api.get_upload_file_headers( upload_auth_token=self.api.account_info.get_account_auth_token(), file_name=self.file_name, content_length=self.size, content_type=self.content_type, content_sha1=self.content_sha1, file_infos=self.file_info, server_side_encryption=sse, file_retention=self.file_retention, legal_hold=self.legal_hold, ) headers_str = ''.join( f'{key}{value}' for key, value in headers.items() if value is not None ) return headers_str.encode('utf8') @property def has_large_header(self) -> bool: """ Determine whether FileVersion's info fits header size limit defined by B2. This function makes sense only for "advanced" buckets, i.e. those which have Server-Side Encryption or File Lock enabled. See https://www.backblaze.com/b2/docs/files.html#httpHeaderSizeLimit. """ return len(self._get_upload_headers()) > self.ADVANCED_HEADERS_LIMIT class DownloadVersion(BaseFileVersion): """ A structure which represents metadata of an initialized download """ __slots__ = [ 'range_', 'content_disposition', 'content_length', 'content_language', '_expires', '_cache_control', 'content_encoding', ] def __init__( self, api: 'B2Api', id_: str, file_name: str, size: int, content_type: Optional[str], content_sha1: Optional[str], file_info: Dict[str, str], upload_timestamp: int, server_side_encryption: EncryptionSetting, range_: Range, content_disposition: Optional[str], content_length: int, content_language: Optional[str], expires, cache_control, content_encoding: Optional[str], file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, replication_status: Optional[ReplicationStatus] = None, ): self.range_ = range_ self.content_disposition = content_disposition self.content_length = content_length self.content_language = content_language self._expires = expires # TODO: parse the string representation of this timestamp to datetime in DownloadVersionFactory self._cache_control = cache_control # TODO: parse the string representation of this mapping to dict in DownloadVersionFactory self.content_encoding = content_encoding super().__init__( api=api, id_=id_, file_name=file_name, size=size, content_type=content_type, content_sha1=content_sha1, file_info=file_info, upload_timestamp=upload_timestamp, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, replication_status=replication_status, ) def _get_args_for_clone(self): args = super()._get_args_for_clone() args.update( { 'range_': self.range_, 'content_disposition': self.content_disposition, 'content_length': self.content_length, 'content_language': self.content_language, 'expires': self._expires, 'cache_control': self._cache_control, 'content_encoding': self.content_encoding, } ) return args class FileVersionFactory: """ Construct :py:class:`b2sdk.v2.FileVersion` objects from api responses. """ FILE_VERSION_CLASS = FileVersion def __init__(self, api: 'B2Api'): self.api = api def from_api_response(self, file_version_dict, force_action=None): """ Turn this: .. code-block:: python { "action": "hide", "fileId": "4_zBucketName_f103b7ca31313c69c_d20151230_m030117_c001_v0001015_t0000", "fileName": "randomdata", "size": 0, "uploadTimestamp": 1451444477000, "replicationStatus": "pending" } or this: .. code-block:: python { "accountId": "4aa9865d6f00", "bucketId": "547a2a395826655d561f0010", "contentLength": 1350, "contentSha1": "753ca1c2d0f3e8748320b38f5da057767029a036", "contentType": "application/octet-stream", "fileId": "4_z547a2a395826655d561f0010_f106d4ca95f8b5b78_d20160104_m003906_c001_v0001013_t0005", "fileInfo": {}, "fileName": "randomdata", "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"}, "replicationStatus": "completed" } into a :py:class:`b2sdk.v2.FileVersion` object. """ assert file_version_dict.get('action') is None or force_action is None, \ 'action was provided by both info_dict and function argument' action = file_version_dict.get('action') or force_action file_name = file_version_dict['fileName'] id_ = file_version_dict['fileId'] if 'size' in file_version_dict: size = file_version_dict['size'] elif 'contentLength' in file_version_dict: size = file_version_dict['contentLength'] else: raise ValueError('no size or contentLength') upload_timestamp = file_version_dict.get('uploadTimestamp') content_type = file_version_dict.get('contentType') content_sha1 = file_version_dict.get('contentSha1') content_md5 = file_version_dict.get('contentMd5') file_info = file_version_dict.get('fileInfo') server_side_encryption = EncryptionSettingFactory.from_file_version_dict(file_version_dict) file_retention = FileRetentionSetting.from_file_version_dict(file_version_dict) legal_hold = LegalHold.from_file_version_dict(file_version_dict) replication_status_value = file_version_dict.get('replicationStatus') replication_status = replication_status_value and ReplicationStatus[ replication_status_value.upper()] return self.FILE_VERSION_CLASS( self.api, id_, file_name, size, content_type, content_sha1, file_info, upload_timestamp, file_version_dict['accountId'], file_version_dict['bucketId'], action, content_md5, server_side_encryption, file_retention, legal_hold, replication_status, ) class DownloadVersionFactory: """ Construct :py:class:`b2sdk.v2.DownloadVersion` objects from download headers. """ def __init__(self, api: 'B2Api'): self.api = api @classmethod def range_and_size_from_header(cls, header): raw_range, raw_size = header.split('/') range_ = Range.from_header(raw_range) size = int(raw_size) return range_, size @classmethod def file_info_from_headers(cls, headers: dict) -> dict: file_info = {} prefix_len = len(FILE_INFO_HEADER_PREFIX_LOWER) for header_name, header_value in headers.items(): if header_name[:prefix_len].lower() == FILE_INFO_HEADER_PREFIX_LOWER: file_info_key = header_name[prefix_len:] file_info[file_info_key] = b2_url_decode(header_value) return file_info def from_response_headers(self, headers): file_info = self.file_info_from_headers(headers) if 'Content-Range' in headers: range_, size = self.range_and_size_from_header(headers['Content-Range']) content_length = int(headers['Content-Length']) else: size = content_length = int(headers['Content-Length']) range_ = Range(0, max(size - 1, 0)) return DownloadVersion( api=self.api, id_=headers['x-bz-file-id'], file_name=b2_url_decode(headers['x-bz-file-name']), size=size, content_type=headers['content-type'], content_sha1=headers['x-bz-content-sha1'], file_info=file_info, upload_timestamp=int(headers['x-bz-upload-timestamp']), server_side_encryption=EncryptionSettingFactory.from_response_headers(headers), range_=range_, content_disposition=headers.get('Content-Disposition'), content_length=content_length, content_language=headers.get('Content-Language'), expires=headers.get('Expires'), cache_control=headers.get('Cache-Control'), content_encoding=headers.get('Content-Encoding'), file_retention=FileRetentionSetting.from_response_headers(headers), legal_hold=LegalHold.from_response_headers(headers), replication_status=ReplicationStatus.from_response_headers(headers), ) class FileIdAndName: """ A structure which represents a B2 cloud file with just `file_name` and `fileId` attributes. Used to return data from calls to b2_delete_file_version and b2_cancel_large_file. """ def __init__(self, file_id: str, file_name: str): self.file_id = file_id self.file_name = file_name @classmethod def from_cancel_or_delete_response(cls, response): return cls(response['fileId'], response['fileName']) def as_dict(self): """ represents the object as a dict which looks almost exactly like the raw api output for delete_file_version """ return {'action': 'delete', 'fileId': self.file_id, 'fileName': self.file_name} def __eq__(self, other): return (self.file_id == other.file_id and self.file_name == other.file_name) def __repr__(self): return '%s(%s, %s)' % (self.__class__.__name__, repr(self.file_id), repr(self.file_name)) b2-sdk-python-1.17.3/b2sdk/http_constants.py000066400000000000000000000015721426424117700206530ustar00rootroot00000000000000###################################################################### # # File: b2sdk/http_constants.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # These constants are needed in different modules, so they are stored in this module, that # imports nothing, thus avoiding circular imports FILE_INFO_HEADER_PREFIX = 'X-Bz-Info-' FILE_INFO_HEADER_PREFIX_LOWER = FILE_INFO_HEADER_PREFIX.lower() # Standard names for file info entries SRC_LAST_MODIFIED_MILLIS = 'src_last_modified_millis' # Special X-Bz-Content-Sha1 value to verify checksum at the end HEX_DIGITS_AT_END = 'hex_digits_at_end' # Identifying SSE_C keys SSE_C_KEY_ID_FILE_INFO_KEY_NAME = 'sse_c_key_id' SSE_C_KEY_ID_HEADER = FILE_INFO_HEADER_PREFIX + SSE_C_KEY_ID_FILE_INFO_KEY_NAME b2-sdk-python-1.17.3/b2sdk/included_sources.py000066400000000000000000000015251426424117700211300ustar00rootroot00000000000000###################################################################### # # File: b2sdk/included_sources.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # This module provides a list of third party sources included and modified in b2sdk, so it can be exposed to # B2 Command Line Tool for printing, for legal compliance reasons import dataclasses from typing import Dict, List _included_sources: List['IncludedSourceMeta'] = [] @dataclasses.dataclass class IncludedSourceMeta: name: str comment: str files: Dict[str, str] def add_included_source(src: IncludedSourceMeta): _included_sources.append(src) def get_included_sources() -> List['IncludedSourceMeta']: return _included_sources b2-sdk-python-1.17.3/b2sdk/large_file/000077500000000000000000000000001426424117700173125ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/large_file/__init__.py000066400000000000000000000004471426424117700214300ustar00rootroot00000000000000###################################################################### # # File: b2sdk/large_file/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/large_file/part.py000066400000000000000000000026471426424117700206430ustar00rootroot00000000000000###################################################################### # # File: b2sdk/large_file/part.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### class PartFactory: @classmethod def from_list_parts_dict(cls, part_dict): return Part( part_dict['fileId'], part_dict['partNumber'], part_dict['contentLength'], part_dict['contentSha1'], ) class Part: """ A structure which represents a *part* of a large file upload. :ivar str ~.file_id: ``fileId`` :ivar int ~.part_number: part number, starting with 1 :ivar str ~.content_length: content length, in bytes :ivar str ~.content_sha1: checksum """ def __init__(self, file_id, part_number, content_length, content_sha1): self.file_id = file_id self.part_number = part_number self.content_length = content_length self.content_sha1 = content_sha1 def __repr__(self): return '<%s %s %s %s %s>' % ( self.__class__.__name__, self.file_id, self.part_number, self.content_length, self.content_sha1 ) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) b2-sdk-python-1.17.3/b2sdk/large_file/services.py000066400000000000000000000110201426424117700215010ustar00rootroot00000000000000###################################################################### # # File: b2sdk/large_file/services.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional from b2sdk.encryption.setting import EncryptionSetting from b2sdk.file_lock import FileRetentionSetting, LegalHold from b2sdk.file_version import FileIdAndName from b2sdk.large_file.part import PartFactory from b2sdk.large_file.unfinished_large_file import UnfinishedLargeFile class LargeFileServices: def __init__(self, services): self.services = services def list_parts(self, file_id, start_part_number=None, batch_size=None): """ Generator that yields a :py:class:`b2sdk.v2.Part` for each of the parts that have been uploaded. :param str file_id: the ID of the large file that is not finished :param int start_part_number: the first part number to return; defaults to the first part :param int batch_size: the number of parts to fetch at a time from the server :rtype: generator """ batch_size = batch_size or 100 while True: response = self.services.session.list_parts(file_id, start_part_number, batch_size) for part_dict in response['parts']: yield PartFactory.from_list_parts_dict(part_dict) start_part_number = response.get('nextPartNumber') if start_part_number is None: break def list_unfinished_large_files( self, bucket_id, start_file_id=None, batch_size=None, prefix=None ): """ A generator that yields an :py:class:`b2sdk.v2.UnfinishedLargeFile` for each unfinished large file in the bucket, starting at the given file, filtering by prefix. :param str bucket_id: bucket id :param str,None start_file_id: a file ID to start from or None to start from the beginning :param int,None batch_size: max file count :param str,None prefix: file name prefix filter :rtype: generator[b2sdk.v2.UnfinishedLargeFile] """ batch_size = batch_size or 100 while True: batch = self.services.session.list_unfinished_large_files( bucket_id, start_file_id, batch_size, prefix ) for file_dict in batch['files']: yield UnfinishedLargeFile(file_dict) start_file_id = batch.get('nextFileId') if start_file_id is None: break def get_unfinished_large_file(self, bucket_id, large_file_id, prefix=None): result = list( self.list_unfinished_large_files( bucket_id, start_file_id=large_file_id, batch_size=1, prefix=prefix ) ) if not result: return None unfinished_large_file = result[0] if unfinished_large_file.file_id != large_file_id: return None return unfinished_large_file def start_large_file( self, bucket_id, file_name, content_type=None, file_info=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): """ Start a large file transfer. :param str file_name: a file name :param str,None content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param dict,None file_info: a file info to store with the file or ``None`` to not store anything :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.LegalHold legal_hold: legal hold setting :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting """ return UnfinishedLargeFile( self.services.session.start_large_file( bucket_id, file_name, content_type, file_info, server_side_encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) ) # delete/cancel def cancel_large_file(self, file_id: str) -> FileIdAndName: """ Cancel a large file upload. """ response = self.services.session.cancel_large_file(file_id) return FileIdAndName.from_cancel_or_delete_response(response) b2-sdk-python-1.17.3/b2sdk/large_file/unfinished_large_file.py000066400000000000000000000034571426424117700242020ustar00rootroot00000000000000###################################################################### # # File: b2sdk/large_file/unfinished_large_file.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.encryption.setting import EncryptionSettingFactory from b2sdk.file_lock import FileRetentionSetting, LegalHold class UnfinishedLargeFile: """ A structure which represents a version of a file (in B2 cloud). :ivar str ~.file_id: ``fileId`` :ivar str ~.file_name: full file name (with path) :ivar str ~.account_id: account ID :ivar str ~.bucket_id: bucket ID :ivar str ~.content_type: :rfc:`822` content type, for example ``"application/octet-stream"`` :ivar dict ~.file_info: file info dict """ def __init__(self, file_dict): """ Initialize from one file returned by ``b2_start_large_file`` or ``b2_list_unfinished_large_files``. """ self.file_id = file_dict['fileId'] self.file_name = file_dict['fileName'] self.account_id = file_dict['accountId'] self.bucket_id = file_dict['bucketId'] self.content_type = file_dict['contentType'] self.file_info = file_dict['fileInfo'] self.encryption = EncryptionSettingFactory.from_file_version_dict(file_dict) self.file_retention = FileRetentionSetting.from_file_version_dict(file_dict) self.legal_hold = LegalHold.from_file_version_dict(file_dict) def __repr__(self): return '<%s %s %s>' % (self.__class__.__name__, self.bucket_id, self.file_name) def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) b2-sdk-python-1.17.3/b2sdk/progress.py000066400000000000000000000132201426424117700174350ustar00rootroot00000000000000###################################################################### # # File: b2sdk/progress.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import ABCMeta, abstractmethod import time try: from tqdm import tqdm # displays a nice progress bar except ImportError: tqdm = None # noqa class AbstractProgressListener(metaclass=ABCMeta): """ Interface expected by B2Api upload and download methods to report on progress. This interface just accepts the number of bytes transferred so far. Subclasses will need to know the total size if they want to report a percent done. """ def __init__(self): self._closed = False @abstractmethod def set_total_bytes(self, total_byte_count): """ Always called before __enter__ to set the expected total number of bytes. May be called more than once if an upload is retried. :param int total_byte_count: expected total number of bytes """ @abstractmethod def bytes_completed(self, byte_count): """ Report the given number of bytes that have been transferred so far. This is not a delta, it is the total number of bytes transferred so far. Transfer can fail and restart from beginning so byte count can decrease between calls. :param int byte_count: number of bytes have been transferred """ def close(self): """ Must be called when you're done with the listener. In well-structured code, should be called only once. """ #import traceback, sys; traceback.print_stack(file=sys.stdout) assert self._closed is False, 'progress listener was closed twice! uncomment the line above to debug this' self._closed = True def __enter__(self): """ A standard context manager method. """ return self def __exit__(self, exc_type, exc_val, exc_tb): """ A standard context manager method. """ self.close() class TqdmProgressListener(AbstractProgressListener): """ Progress listener based on tqdm library. """ def __init__(self, description, *args, **kwargs): self.description = description self.tqdm = None # set in set_total_bytes() self.prev_value = 0 super(TqdmProgressListener, self).__init__(*args, **kwargs) def set_total_bytes(self, total_byte_count): if self.tqdm is None: self.tqdm = tqdm( desc=self.description, total=total_byte_count, unit='B', unit_scale=True, leave=True, miniters=1, smoothing=0.1, mininterval=0.2, ) def bytes_completed(self, byte_count): # tqdm doesn't support running the progress bar backwards, # so on an upload retry, it just won't move until it gets # past the point where it failed. if self.prev_value < byte_count: self.tqdm.update(byte_count - self.prev_value) self.prev_value = byte_count def close(self): if self.tqdm is not None: self.tqdm.close() super(TqdmProgressListener, self).close() class SimpleProgressListener(AbstractProgressListener): """ Just a simple progress listener which prints info on a console. """ def __init__(self, description, *args, **kwargs): self.desc = description self.complete = 0 self.last_time = time.time() self.any_printed = False super(SimpleProgressListener, self).__init__(*args, **kwargs) def set_total_bytes(self, total_byte_count): self.total = total_byte_count def bytes_completed(self, byte_count): now = time.time() elapsed = now - self.last_time if 3 <= elapsed and self.total != 0: if not self.any_printed: print(self.desc) print(' %d%%' % int(100.0 * byte_count / self.total)) self.last_time = now self.any_printed = True def close(self): if self.any_printed: print(' DONE.') super(SimpleProgressListener, self).close() class DoNothingProgressListener(AbstractProgressListener): """ This listener gives no output whatsoever. """ def set_total_bytes(self, total_byte_count): pass def bytes_completed(self, byte_count): pass class ProgressListenerForTest(AbstractProgressListener): """ Capture all of the calls so they can be checked. """ def __init__(self, *args, **kwargs): self.calls = [] super(ProgressListenerForTest, self).__init__(*args, **kwargs) def set_total_bytes(self, total_byte_count): self.calls.append('set_total_bytes(%d)' % (total_byte_count,)) def bytes_completed(self, byte_count): self.calls.append('bytes_completed(%d)' % (byte_count,)) def close(self): self.calls.append('close()') super(ProgressListenerForTest, self).close() def get_calls(self): return self.calls def make_progress_listener(description, quiet): """ Return a progress listener object depending on some conditions. :param str description: listener description :param bool quiet: if ``True``, do not output anything :return: a listener object """ if quiet: return DoNothingProgressListener() elif tqdm is not None: return TqdmProgressListener(description) else: return SimpleProgressListener(description) b2-sdk-python-1.17.3/b2sdk/raw_api.py000066400000000000000000001030741426424117700172220ustar00rootroot00000000000000###################################################################### # # File: b2sdk/raw_api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import base64 import re from abc import ABCMeta, abstractmethod from enum import Enum, unique from logging import getLogger from typing import Any, Dict, Optional from .exception import FileOrBucketNotFound, ResourceNotFound, UnusableFileName, InvalidMetadataDirective, WrongEncryptionModeForBucketDefault, AccessDenied, SSECKeyError, RetentionWriteError from .encryption.setting import EncryptionMode, EncryptionSetting from .replication.setting import ReplicationConfiguration from .file_lock import BucketRetentionSetting, FileRetentionSetting, LegalHold from .utils import b2_url_encode from b2sdk.http_constants import FILE_INFO_HEADER_PREFIX # All supported realms REALM_URLS = { 'production': 'https://api.backblazeb2.com', 'dev': 'http://api.backblazeb2.xyz:8180', 'staging': 'https://api.backblaze.net', } # All possible capabilities ALL_CAPABILITIES = [ 'listKeys', 'writeKeys', 'deleteKeys', 'listBuckets', 'listAllBucketNames', 'readBuckets', 'writeBuckets', 'deleteBuckets', 'readBucketEncryption', 'writeBucketEncryption', 'readBucketRetentions', 'writeBucketRetentions', 'readFileRetentions', 'writeFileRetentions', 'readFileLegalHolds', 'writeFileLegalHolds', 'readBucketReplications', 'writeBucketReplications', 'bypassGovernance', 'listFiles', 'readFiles', 'shareFiles', 'writeFiles', 'deleteFiles', ] # API version number to use when calling the service API_VERSION = 'v2' logger = getLogger(__name__) @unique class MetadataDirectiveMode(Enum): """ Mode of handling metadata when copying a file """ COPY = 401 #: copy metadata from the source file REPLACE = 402 #: ignore the source file metadata and set it to provided values class AbstractRawApi(metaclass=ABCMeta): """ Direct access to the B2 web apis. """ @abstractmethod def authorize_account(self, realm_url, application_key_id, application_key): pass @abstractmethod def cancel_large_file(self, api_url, account_auth_token, file_id): pass @abstractmethod def copy_file( self, api_url, account_auth_token, source_file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_bucket_id=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): pass @abstractmethod def copy_part( self, api_url, account_auth_token, source_file_id, large_file_id, part_number, bytes_range=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, ): pass @abstractmethod def create_bucket( self, api_url, account_auth_token, account_id, bucket_name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, is_file_lock_enabled: Optional[bool] = None, replication: Optional[ReplicationConfiguration] = None, ): pass @abstractmethod def create_key( self, api_url, account_auth_token, account_id, capabilities, key_name, valid_duration_seconds, bucket_id, name_prefix ): pass @abstractmethod def download_file_from_url( self, account_auth_token_or_none, url, range_=None, encryption: Optional[EncryptionSetting] = None, ): pass @abstractmethod def delete_key(self, api_url, account_auth_token, application_key_id): pass @abstractmethod def delete_bucket(self, api_url, account_auth_token, account_id, bucket_id): pass @abstractmethod def delete_file_version(self, api_url, account_auth_token, file_id, file_name): pass @abstractmethod def finish_large_file(self, api_url, account_auth_token, file_id, part_sha1_array): pass @abstractmethod def get_download_authorization( self, api_url, account_auth_token, bucket_id, file_name_prefix, valid_duration_in_seconds ): pass @abstractmethod def get_file_info_by_id(self, api_url: str, account_auth_token: str, file_id: str) -> Dict[str, Any]: pass @abstractmethod def get_file_info_by_name( self, download_url: str, account_auth_token: str, bucket_name: str, file_name: str ) -> Dict[str, Any]: pass @abstractmethod def get_upload_url(self, api_url, account_auth_token, bucket_id): pass @abstractmethod def get_upload_part_url(self, api_url, account_auth_token, file_id): pass @abstractmethod def hide_file(self, api_url, account_auth_token, bucket_id, file_name): pass @abstractmethod def list_buckets( self, api_url, account_auth_token, account_id, bucket_id=None, bucket_name=None, ): pass @abstractmethod def list_file_names( self, api_url, account_auth_token, bucket_id, start_file_name=None, max_file_count=None, prefix=None, ): pass @abstractmethod def list_file_versions( self, api_url, account_auth_token, bucket_id, start_file_name=None, start_file_id=None, max_file_count=None, prefix=None, ): pass @abstractmethod def list_keys( self, api_url, account_auth_token, account_id, max_key_count=None, start_application_key_id=None ): pass @abstractmethod def list_parts(self, api_url, account_auth_token, file_id, start_part_number, max_part_count): pass @abstractmethod def list_unfinished_large_files( self, api_url, account_auth_token, bucket_id, start_file_id=None, max_file_count=None, prefix=None, ): pass @abstractmethod def start_large_file( self, api_url, account_auth_token, bucket_id, file_name, content_type, file_info, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): pass @abstractmethod def update_bucket( self, api_url, account_auth_token, account_id, bucket_id, bucket_type=None, bucket_info=None, cors_rules=None, lifecycle_rules=None, if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, replication: Optional[ReplicationConfiguration] = None, ): pass @abstractmethod def update_file_retention( self, api_url, account_auth_token, file_id, file_name, file_retention: FileRetentionSetting, bypass_governance: bool = False, ): pass @classmethod def get_upload_file_headers( cls, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_infos: dict, server_side_encryption: Optional[EncryptionSetting], file_retention: Optional[FileRetentionSetting], legal_hold: Optional[LegalHold], ) -> dict: headers = { 'Authorization': upload_auth_token, 'Content-Length': str(content_length), 'X-Bz-File-Name': b2_url_encode(file_name), 'Content-Type': content_type, 'X-Bz-Content-Sha1': content_sha1, } for k, v in file_infos.items(): headers[FILE_INFO_HEADER_PREFIX + k] = b2_url_encode(v) if server_side_encryption is not None: assert server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C ) server_side_encryption.add_to_upload_headers(headers) if legal_hold is not None: legal_hold.add_to_upload_headers(headers) if file_retention is not None: file_retention.add_to_to_upload_headers(headers) return headers @abstractmethod def upload_file( self, upload_url, upload_auth_token, file_name, content_length, content_type, content_sha1, file_infos, data_stream, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): pass @abstractmethod def upload_part( self, upload_url, upload_auth_token, part_number, content_length, sha1_sum, input_stream, server_side_encryption: Optional[EncryptionSetting] = None, ): pass def get_download_url_by_id(self, download_url, file_id): return '%s/b2api/%s/b2_download_file_by_id?fileId=%s' % (download_url, API_VERSION, file_id) def get_download_url_by_name(self, download_url, bucket_name, file_name): return download_url + '/file/' + bucket_name + '/' + b2_url_encode(file_name) class B2RawHTTPApi(AbstractRawApi): """ Provide access to the B2 web APIs, exactly as they are provided by b2. Requires that you provide all necessary URLs and auth tokens for each call. Each API call decodes the returned JSON and returns a dict. For details on what each method does, see the B2 docs: https://www.backblaze.com/b2/docs/ This class is intended to be a super-simple, very thin layer on top of the HTTP calls. It can be mocked-out for testing higher layers. And this class can be tested by exercising each call just once, which is relatively quick. """ def __init__(self, b2_http): self.b2_http = b2_http def _post_json(self, base_url, api_name, auth, **params) -> Dict[str, Any]: """ A helper method for calling an API with the given auth and params. :param base_url: something like "https://api001.backblazeb2.com/" :param auth: passed in Authorization header :param api_name: example: "b2_create_bucket" :param args: the rest of the parameters are passed to b2 :return: the decoded JSON response :rtype: dict """ url = '%s/b2api/%s/%s' % (base_url, API_VERSION, api_name) headers = {'Authorization': auth} return self.b2_http.post_json_return_json(url, headers, params) def authorize_account(self, realm_url, application_key_id, application_key): auth = b'Basic ' + base64.b64encode( ('%s:%s' % (application_key_id, application_key)).encode() ) return self._post_json(realm_url, 'b2_authorize_account', auth) def cancel_large_file(self, api_url, account_auth_token, file_id): return self._post_json(api_url, 'b2_cancel_large_file', account_auth_token, fileId=file_id) def create_bucket( self, api_url, account_auth_token, account_id, bucket_name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, is_file_lock_enabled: Optional[bool] = None, replication: Optional[ReplicationConfiguration] = None, ): kwargs = dict( accountId=account_id, bucketName=bucket_name, bucketType=bucket_type, ) if bucket_info is not None: kwargs['bucketInfo'] = bucket_info if cors_rules is not None: kwargs['corsRules'] = cors_rules if lifecycle_rules is not None: kwargs['lifecycleRules'] = lifecycle_rules if default_server_side_encryption is not None: if not default_server_side_encryption.mode.can_be_set_as_bucket_default(): raise WrongEncryptionModeForBucketDefault(default_server_side_encryption.mode) kwargs['defaultServerSideEncryption' ] = default_server_side_encryption.serialize_to_json_for_request() if is_file_lock_enabled is not None: kwargs['fileLockEnabled'] = is_file_lock_enabled if replication is not None: kwargs['replicationConfiguration'] = replication.serialize_to_json_for_request() return self._post_json( api_url, 'b2_create_bucket', account_auth_token, **kwargs, ) def create_key( self, api_url, account_auth_token, account_id, capabilities, key_name, valid_duration_seconds, bucket_id, name_prefix ): return self._post_json( api_url, 'b2_create_key', account_auth_token, accountId=account_id, capabilities=capabilities, keyName=key_name, validDurationInSeconds=valid_duration_seconds, bucketId=bucket_id, namePrefix=name_prefix, ) def delete_bucket(self, api_url, account_auth_token, account_id, bucket_id): return self._post_json( api_url, 'b2_delete_bucket', account_auth_token, accountId=account_id, bucketId=bucket_id ) def delete_file_version(self, api_url, account_auth_token, file_id, file_name): return self._post_json( api_url, 'b2_delete_file_version', account_auth_token, fileId=file_id, fileName=file_name ) def delete_key(self, api_url, account_auth_token, application_key_id): return self._post_json( api_url, 'b2_delete_key', account_auth_token, applicationKeyId=application_key_id, ) def download_file_from_url( self, account_auth_token_or_none, url, range_=None, encryption: Optional[EncryptionSetting] = None, ): """ Issue a streaming request for download of a file, potentially authorized. :param str account_auth_token_or_none: an optional account auth token to pass in :param str url: the full URL to download from :param tuple range: two-element tuple for http Range header :param b2sdk.v2.EncryptionSetting encryption: encryption settings for downloading :return: b2_http response """ request_headers = {} _add_range_header(request_headers, range_) if encryption is not None: assert encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C ) encryption.add_to_download_headers(request_headers) if account_auth_token_or_none is not None: request_headers['Authorization'] = account_auth_token_or_none try: return self.b2_http.get_content(url, request_headers) except AccessDenied: raise SSECKeyError() def finish_large_file(self, api_url, account_auth_token, file_id, part_sha1_array): return self._post_json( api_url, 'b2_finish_large_file', account_auth_token, fileId=file_id, partSha1Array=part_sha1_array ) def get_download_authorization( self, api_url, account_auth_token, bucket_id, file_name_prefix, valid_duration_in_seconds ): return self._post_json( api_url, 'b2_get_download_authorization', account_auth_token, bucketId=bucket_id, fileNamePrefix=file_name_prefix, validDurationInSeconds=valid_duration_in_seconds ) def get_file_info_by_id(self, api_url: str, account_auth_token: str, file_id: str) -> Dict[str, Any]: return self._post_json(api_url, 'b2_get_file_info', account_auth_token, fileId=file_id) def get_file_info_by_name( self, download_url: str, account_auth_token: str, bucket_name: str, file_name: str ) -> Dict[str, Any]: download_url = self.get_download_url_by_name(download_url, bucket_name, file_name) try: response = self.b2_http.head_content( download_url, headers={"Authorization": account_auth_token} ) return response.headers except ResourceNotFound: logger.debug("Resource Not Found: %s" % download_url) raise FileOrBucketNotFound(bucket_name, file_name) def get_upload_url(self, api_url, account_auth_token, bucket_id): return self._post_json(api_url, 'b2_get_upload_url', account_auth_token, bucketId=bucket_id) def get_upload_part_url(self, api_url, account_auth_token, file_id): return self._post_json( api_url, 'b2_get_upload_part_url', account_auth_token, fileId=file_id ) def hide_file(self, api_url, account_auth_token, bucket_id, file_name): return self._post_json( api_url, 'b2_hide_file', account_auth_token, bucketId=bucket_id, fileName=file_name ) def list_buckets( self, api_url, account_auth_token, account_id, bucket_id=None, bucket_name=None, ): return self._post_json( api_url, 'b2_list_buckets', account_auth_token, accountId=account_id, bucketTypes=['all'], bucketId=bucket_id, bucketName=bucket_name, ) def list_file_names( self, api_url, account_auth_token, bucket_id, start_file_name=None, max_file_count=None, prefix=None, ): return self._post_json( api_url, 'b2_list_file_names', account_auth_token, bucketId=bucket_id, startFileName=start_file_name, maxFileCount=max_file_count, prefix=prefix, ) def list_file_versions( self, api_url, account_auth_token, bucket_id, start_file_name=None, start_file_id=None, max_file_count=None, prefix=None, ): return self._post_json( api_url, 'b2_list_file_versions', account_auth_token, bucketId=bucket_id, startFileName=start_file_name, startFileId=start_file_id, maxFileCount=max_file_count, prefix=prefix, ) def list_keys( self, api_url, account_auth_token, account_id, max_key_count=None, start_application_key_id=None ): return self._post_json( api_url, 'b2_list_keys', account_auth_token, accountId=account_id, maxKeyCount=max_key_count, startApplicationKeyId=start_application_key_id, ) def list_parts(self, api_url, account_auth_token, file_id, start_part_number, max_part_count): return self._post_json( api_url, 'b2_list_parts', account_auth_token, fileId=file_id, startPartNumber=start_part_number, maxPartCount=max_part_count ) def list_unfinished_large_files( self, api_url, account_auth_token, bucket_id, start_file_id=None, max_file_count=None, prefix=None, ): return self._post_json( api_url, 'b2_list_unfinished_large_files', account_auth_token, bucketId=bucket_id, startFileId=start_file_id, maxFileCount=max_file_count, namePrefix=prefix, ) def start_large_file( self, api_url, account_auth_token, bucket_id, file_name, content_type, file_info, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): kwargs = {} if server_side_encryption is not None: assert server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C ) kwargs['serverSideEncryption'] = server_side_encryption.serialize_to_json_for_request() if legal_hold is not None: kwargs['legalHold'] = legal_hold.to_server() if file_retention is not None: kwargs['fileRetention'] = file_retention.serialize_to_json_for_request() return self._post_json( api_url, 'b2_start_large_file', account_auth_token, bucketId=bucket_id, fileName=file_name, fileInfo=file_info, contentType=content_type, **kwargs ) def update_bucket( self, api_url, account_auth_token, account_id, bucket_id, bucket_type=None, bucket_info=None, cors_rules=None, lifecycle_rules=None, if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, replication: Optional[ReplicationConfiguration] = None, ): kwargs = {} if if_revision_is is not None: kwargs['ifRevisionIs'] = if_revision_is if bucket_info is not None: kwargs['bucketInfo'] = bucket_info if bucket_type is not None: kwargs['bucketType'] = bucket_type if cors_rules is not None: kwargs['corsRules'] = cors_rules if lifecycle_rules is not None: kwargs['lifecycleRules'] = lifecycle_rules if default_server_side_encryption is not None: if not default_server_side_encryption.mode.can_be_set_as_bucket_default(): raise WrongEncryptionModeForBucketDefault(default_server_side_encryption.mode) kwargs['defaultServerSideEncryption' ] = default_server_side_encryption.serialize_to_json_for_request() if default_retention is not None: kwargs['defaultRetention'] = default_retention.serialize_to_json_for_request() if replication is not None: kwargs['replicationConfiguration'] = replication.serialize_to_json_for_request() assert kwargs return self._post_json( api_url, 'b2_update_bucket', account_auth_token, accountId=account_id, bucketId=bucket_id, **kwargs ) def update_file_retention( self, api_url, account_auth_token, file_id, file_name, file_retention: FileRetentionSetting, bypass_governance: bool = False, ): kwargs = {} kwargs['fileRetention'] = file_retention.serialize_to_json_for_request() try: return self._post_json( api_url, 'b2_update_file_retention', account_auth_token, fileId=file_id, fileName=file_name, bypassGovernance=bypass_governance, **kwargs ) except AccessDenied: raise RetentionWriteError() def update_file_legal_hold( self, api_url, account_auth_token, file_id, file_name, legal_hold: LegalHold, ): try: return self._post_json( api_url, 'b2_update_file_legal_hold', account_auth_token, fileId=file_id, fileName=file_name, legalHold=legal_hold.to_server(), ) except AccessDenied: raise RetentionWriteError() def unprintable_to_hex(self, string): """ Replace unprintable chars in string with a hex representation. :param string: an arbitrary string, possibly with unprintable characters. :return: the string, with unprintable characters changed to hex (e.g., "\x07") """ unprintables_pattern = re.compile(r'[\x00-\x1f]') def hexify(match): return r'\x{0:02x}'.format(ord(match.group())) return unprintables_pattern.sub(hexify, string) def check_b2_filename(self, filename): """ Raise an appropriate exception with details if the filename is unusable. See https://www.backblaze.com/b2/docs/files.html for the rules. :param filename: a proposed filename in unicode :return: None if the filename is usable """ encoded_name = filename.encode('utf-8') length_in_bytes = len(encoded_name) if length_in_bytes < 1: raise UnusableFileName("Filename must be at least 1 character.") if length_in_bytes > 1024: raise UnusableFileName("Filename is too long (can be at most 1024 bytes).") lowest_unicode_value = ord(min(filename)) if lowest_unicode_value < 32: message = u"Filename \"{0}\" contains code {1} (hex {2:02x}), less than 32.".format( self.unprintable_to_hex(filename), lowest_unicode_value, lowest_unicode_value ) raise UnusableFileName(message) # No DEL for you. if '\x7f' in filename: raise UnusableFileName("DEL character (0x7f) not allowed.") if filename[0] == '/' or filename[-1] == '/': raise UnusableFileName("Filename may not start or end with '/'.") if '//' in filename: raise UnusableFileName("Filename may not contain \"//\".") long_segment = max([len(segment.encode('utf-8')) for segment in filename.split('/')]) if long_segment > 250: raise UnusableFileName("Filename segment too long (maximum 250 bytes in utf-8).") def upload_file( self, upload_url, upload_auth_token, file_name, content_length, content_type, content_sha1, file_infos, data_stream, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): """ Upload one, small file to b2. :param upload_url: the upload_url from b2_authorize_account :param upload_auth_token: the auth token from b2_authorize_account :param file_name: the name of the B2 file :param content_length: number of bytes in the file :param content_type: MIME type :param content_sha1: hex SHA1 of the contents of the file :param file_infos: extra file info to upload :param data_stream: a file like object from which the contents of the file can be read :return: """ # Raise UnusableFileName if the file_name doesn't meet the rules. self.check_b2_filename(file_name) headers = self.get_upload_file_headers( upload_auth_token=upload_auth_token, file_name=file_name, content_length=content_length, content_type=content_type, content_sha1=content_sha1, file_infos=file_infos, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, ) return self.b2_http.post_content_return_json(upload_url, headers, data_stream) def upload_part( self, upload_url, upload_auth_token, part_number, content_length, content_sha1, data_stream, server_side_encryption: Optional[EncryptionSetting] = None, ): headers = { 'Authorization': upload_auth_token, 'Content-Length': str(content_length), 'X-Bz-Part-Number': str(part_number), 'X-Bz-Content-Sha1': content_sha1 } if server_side_encryption is not None: assert server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C ) server_side_encryption.add_to_upload_headers(headers) return self.b2_http.post_content_return_json(upload_url, headers, data_stream) def copy_file( self, api_url, account_auth_token, source_file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_bucket_id=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): kwargs = {} if bytes_range is not None: range_dict = {} _add_range_header(range_dict, bytes_range) kwargs['range'] = range_dict['Range'] if metadata_directive is not None: assert metadata_directive in tuple(MetadataDirectiveMode) if metadata_directive is MetadataDirectiveMode.COPY and ( content_type is not None or file_info is not None ): raise InvalidMetadataDirective( 'content_type and file_info should be None when metadata_directive is COPY' ) elif metadata_directive is MetadataDirectiveMode.REPLACE and content_type is None: raise InvalidMetadataDirective( 'content_type cannot be None when metadata_directive is REPLACE' ) kwargs['metadataDirective'] = metadata_directive.name if content_type is not None: kwargs['contentType'] = content_type if file_info is not None: kwargs['fileInfo'] = file_info if destination_bucket_id is not None: kwargs['destinationBucketId'] = destination_bucket_id if destination_server_side_encryption is not None: assert destination_server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C ) kwargs['destinationServerSideEncryption' ] = destination_server_side_encryption.serialize_to_json_for_request() if source_server_side_encryption is not None: assert source_server_side_encryption.mode == EncryptionMode.SSE_C kwargs['sourceServerSideEncryption' ] = source_server_side_encryption.serialize_to_json_for_request() if legal_hold is not None: kwargs['legalHold'] = legal_hold.to_server() if file_retention is not None: kwargs['fileRetention'] = file_retention.serialize_to_json_for_request() try: return self._post_json( api_url, 'b2_copy_file', account_auth_token, sourceFileId=source_file_id, fileName=new_file_name, **kwargs ) except AccessDenied: raise SSECKeyError() def copy_part( self, api_url, account_auth_token, source_file_id, large_file_id, part_number, bytes_range=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, ): kwargs = {} if bytes_range is not None: range_dict = {} _add_range_header(range_dict, bytes_range) kwargs['range'] = range_dict['Range'] if destination_server_side_encryption is not None: assert destination_server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C ) kwargs['destinationServerSideEncryption' ] = destination_server_side_encryption.serialize_to_json_for_request() if source_server_side_encryption is not None: assert source_server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C ) kwargs['sourceServerSideEncryption' ] = source_server_side_encryption.serialize_to_json_for_request() try: return self._post_json( api_url, 'b2_copy_part', account_auth_token, sourceFileId=source_file_id, largeFileId=large_file_id, partNumber=part_number, **kwargs ) except AccessDenied: raise SSECKeyError() def _add_range_header(headers, range_): if range_ is not None: assert len(range_) == 2, range_ assert (range_[0] + 0) <= (range_[1] + 0), range_ # not strings assert range_[0] >= 0, range_ headers['Range'] = "bytes=%d-%d" % range_ b2-sdk-python-1.17.3/b2sdk/raw_simulator.py000066400000000000000000002177441426424117700205020ustar00rootroot00000000000000###################################################################### # # File: b2sdk/raw_simulator.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import collections import io import logging import random import re import threading import time from contextlib import contextmanager from typing import Optional from b2sdk.http_constants import FILE_INFO_HEADER_PREFIX, HEX_DIGITS_AT_END from b2sdk.replication.setting import ReplicationConfiguration from requests.structures import CaseInsensitiveDict from .b2http import ResponseContextManager from .encryption.setting import EncryptionMode, EncryptionSetting from .replication.types import ReplicationStatus from .exception import ( BadJson, BadRequest, BadUploadUrl, ChecksumMismatch, Conflict, CopySourceTooBig, DuplicateBucketName, FileNotPresent, FileSha1Mismatch, InvalidAuthToken, InvalidMetadataDirective, MissingPart, NonExistentBucket, PartSha1Mismatch, SSECKeyError, Unauthorized, UnsatisfiableRange, ) from .file_lock import ( NO_RETENTION_BUCKET_SETTING, BucketRetentionSetting, FileRetentionSetting, LegalHold, ) from .file_version import UNVERIFIED_CHECKSUM_PREFIX from .raw_api import ALL_CAPABILITIES, AbstractRawApi, MetadataDirectiveMode from .stream.hashing import StreamWithHash from .utils import ConcurrentUsedAuthTokenGuard, b2_url_decode, b2_url_encode, hex_sha1_of_bytes logger = logging.getLogger(__name__) def get_bytes_range(data_bytes, bytes_range): """ Slice bytes array using bytes range """ if bytes_range is None: return data_bytes if bytes_range[0] > bytes_range[1]: raise UnsatisfiableRange() if bytes_range[0] < 0: raise UnsatisfiableRange() if bytes_range[1] > len(data_bytes): raise UnsatisfiableRange() return data_bytes[bytes_range[0]:bytes_range[1] + 1] class KeySimulator: """ Hold information about one application key, which can be either a master application key, or one created with create_key(). """ def __init__( self, account_id, name, application_key_id, key, capabilities, expiration_timestamp_or_none, bucket_id_or_none, bucket_name_or_none, name_prefix_or_none ): self.name = name self.account_id = account_id self.application_key_id = application_key_id self.key = key self.capabilities = capabilities self.expiration_timestamp_or_none = expiration_timestamp_or_none self.bucket_id_or_none = bucket_id_or_none self.name_prefix_or_none = name_prefix_or_none def as_key(self): return dict( accountId=self.account_id, bucketId=self.bucket_id_or_none, applicationKeyId=self.application_key_id, capabilities=self.capabilities, expirationTimestamp=self.expiration_timestamp_or_none and self.expiration_timestamp_or_none * 1000, keyName=self.name, namePrefix=self.name_prefix_or_none, ) def as_created_key(self): """ Return the dict returned by b2_create_key. This is just like the one for b2_list_keys, but also includes the secret key. """ result = self.as_key() result['applicationKey'] = self.key return result def get_allowed(self): """ Return the 'allowed' structure to include in the response from b2_authorize_account. """ return dict( bucketId=self.bucket_id_or_none, capabilities=self.capabilities, namePrefix=self.name_prefix_or_none, ) class PartSimulator: def __init__(self, file_id, part_number, content_length, content_sha1, part_data): self.file_id = file_id self.part_number = part_number self.content_length = content_length self.content_sha1 = content_sha1 self.part_data = part_data def as_list_parts_dict(self): return dict( fileId=self.file_id, partNumber=self.part_number, contentLength=self.content_length, contentSha1=self.content_sha1 ) # yapf: disable class FileSimulator: """ One of three: an unfinished large file, a finished file, or a deletion marker. """ CHECK_ENCRYPTION = True SPECIAL_FILE_INFOS = { # when downloading, these file info keys are translated to specialized headers 'b2-content-disposition': 'Content-Disposition', 'b2-content-language': 'Content-Language', 'b2-expires': 'Expires', 'b2-cache-control': 'Cache-Control', 'b2-content-encoding': 'Content-Encoding', } def __init__( self, account_id, bucket, file_id, action, name, content_type, content_sha1, file_info, data_bytes, upload_timestamp, range_=None, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: LegalHold = LegalHold.UNSET, replication_status: Optional[ReplicationStatus] = None, ): if action == 'hide': assert server_side_encryption is None else: assert server_side_encryption is not None self.account_id = account_id self.bucket = bucket self.file_id = file_id self.action = action self.name = name if data_bytes is not None: self.content_length = len(data_bytes) self.content_type = content_type self.content_sha1 = content_sha1 if content_sha1 and content_sha1 != 'none' and len(content_sha1) != 40: raise ValueError(content_sha1) self.file_info = file_info self.data_bytes = data_bytes self.upload_timestamp = upload_timestamp self.range_ = range_ self.server_side_encryption = server_side_encryption self.file_retention = file_retention self.legal_hold = legal_hold if legal_hold is not None else LegalHold.UNSET self.replication_status = replication_status if action == 'start': self.parts = [] @classmethod @contextmanager def dont_check_encryption(cls): cls.CHECK_ENCRYPTION = False yield cls.CHECK_ENCRYPTION = True def sort_key(self): """ Return a key that can be used to sort the files in a bucket in the order that b2_list_file_versions returns them. """ return (self.name, self.file_id) def as_download_headers(self, account_auth_token_or_none, range_=None): if self.data_bytes is None: content_length = 0 elif range_ is not None: if range_[1] >= len(self.data_bytes): # requested too much content_length = len(self.data_bytes) else: content_length = range_[1] - range_[0] + 1 else: content_length = len(self.data_bytes) headers = CaseInsensitiveDict( { 'content-length': content_length, 'content-type': self.content_type, 'x-bz-content-sha1': self.content_sha1, 'x-bz-upload-timestamp': self.upload_timestamp, 'x-bz-file-id': self.file_id, 'x-bz-file-name': b2_url_encode(self.name), } ) for key, value in self.file_info.items(): key_lower = key.lower() if key_lower in self.SPECIAL_FILE_INFOS: headers[self.SPECIAL_FILE_INFOS[key_lower]] = value else: headers[FILE_INFO_HEADER_PREFIX + key] = b2_url_encode(value) if account_auth_token_or_none is not None and self.bucket.is_file_lock_enabled: not_permitted = [] if not self.is_allowed_to_read_file_retention(account_auth_token_or_none): not_permitted.append('X-Bz-File-Retention-Mode') not_permitted.append('X-Bz-File-Retain-Until-Timestamp') else: if self.file_retention is not None: self.file_retention.add_to_to_upload_headers(headers) if not self.is_allowed_to_read_file_legal_hold(account_auth_token_or_none): not_permitted.append('X-Bz-File-Legal-Hold') else: headers['X-Bz-File-Legal-Hold'] = self.legal_hold and 'on' or 'off' if not_permitted: headers['X-Bz-Client-Unauthorized-To-Read'] = ','.join(not_permitted) if self.server_side_encryption is not None: if self.server_side_encryption.mode == EncryptionMode.SSE_B2: headers['X-Bz-Server-Side-Encryption'] = self.server_side_encryption.algorithm.value elif self.server_side_encryption.mode == EncryptionMode.SSE_C: headers['X-Bz-Server-Side-Encryption-Customer-Algorithm' ] = self.server_side_encryption.algorithm.value headers['X-Bz-Server-Side-Encryption-Customer-Key-Md5' ] = self.server_side_encryption.key.key_md5() elif self.server_side_encryption.mode in (EncryptionMode.NONE, EncryptionMode.UNKNOWN): pass else: raise ValueError( 'Unsupported encryption mode: %s' % (self.server_side_encryption.mode,) ) if range_ is not None: headers['Content-Range'] = 'bytes %d-%d/%d' % ( range_[0], range_[0] + content_length - 1, len(self.data_bytes) ) # yapf: disable return headers def as_upload_result(self, account_auth_token): result = dict( fileId=self.file_id, fileName=self.name, accountId=self.account_id, bucketId=self.bucket.bucket_id, contentLength=len(self.data_bytes) if self.data_bytes is not None else 0, contentType=self.content_type, contentSha1=self.content_sha1, fileInfo=self.file_info, action=self.action, uploadTimestamp=self.upload_timestamp, replicationStatus=self.replication_status and self.replication_status.value, ) # yapf: disable if self.server_side_encryption is not None: result['serverSideEncryption' ] = self.server_side_encryption.serialize_to_json_for_request() result['fileRetention'] = self._file_retention_dict(account_auth_token) result['legalHold'] = self._legal_hold_dict(account_auth_token) return result def as_list_files_dict(self, account_auth_token): result = dict( accountId=self.account_id, bucketId=self.bucket.bucket_id, fileId=self.file_id, fileName=self.name, contentLength=len(self.data_bytes) if self.data_bytes is not None else 0, contentType=self.content_type, contentSha1=self.content_sha1, fileInfo=self.file_info, action=self.action, uploadTimestamp=self.upload_timestamp, replicationStatus=self.replication_status and self.replication_status.value, ) # yapf: disable if self.server_side_encryption is not None: result['serverSideEncryption' ] = self.server_side_encryption.serialize_to_json_for_request() result['fileRetention'] = self._file_retention_dict(account_auth_token) result['legalHold'] = self._legal_hold_dict(account_auth_token) return result def is_allowed_to_read_file_retention(self, account_auth_token): return self.bucket._check_capability(account_auth_token, 'readFileRetentions') def is_allowed_to_read_file_legal_hold(self, account_auth_token): return self.bucket._check_capability(account_auth_token, 'readFileLegalHolds') def as_start_large_file_result(self, account_auth_token): result = dict( fileId=self.file_id, fileName=self.name, accountId=self.account_id, bucketId=self.bucket.bucket_id, contentType=self.content_type, fileInfo=self.file_info, uploadTimestamp=self.upload_timestamp, replicationStatus=self.replication_status and self.replication_status.value, ) # yapf: disable if self.server_side_encryption is not None: result['serverSideEncryption' ] = self.server_side_encryption.serialize_to_json_for_request() result['fileRetention'] = self._file_retention_dict(account_auth_token) result['legalHold'] = self._legal_hold_dict(account_auth_token) return result def _file_retention_dict(self, account_auth_token): if not self.is_allowed_to_read_file_retention(account_auth_token): return { 'isClientAuthorizedToRead': False, 'value': None, } file_lock_configuration = {'isClientAuthorizedToRead': True} if self.file_retention is None: file_lock_configuration['value'] = {'mode': None} else: file_lock_configuration['value'] = {'mode': self.file_retention.mode.value} if self.file_retention.retain_until is not None: file_lock_configuration['value']['retainUntilTimestamp' ] = self.file_retention.retain_until return file_lock_configuration def _legal_hold_dict(self, account_auth_token): if not self.is_allowed_to_read_file_legal_hold(account_auth_token): return { 'isClientAuthorizedToRead': False, 'value': None, } return { 'isClientAuthorizedToRead': True, 'value': self.legal_hold.value, } def add_part(self, part_number, part): while len(self.parts) < part_number + 1: self.parts.append(None) self.parts[part_number] = part def finish(self, part_sha1_array): last_part_number = max(part.part_number for part in self.parts if part is not None) for part_number in range(1, last_part_number + 1): if self.parts[part_number] is None: raise MissingPart(part_number) my_part_sha1_array = [ self.parts[part_number].content_sha1 for part_number in range(1, last_part_number + 1) ] if part_sha1_array != my_part_sha1_array: raise ChecksumMismatch( 'sha1', expected=str(part_sha1_array), actual=str(my_part_sha1_array) ) self.data_bytes = b''.join( self.parts[part_number].part_data for part_number in range(1, last_part_number + 1) ) self.content_length = len(self.data_bytes) self.action = 'upload' def is_visible(self): """ Does this file show up in b2_list_file_names? """ return self.action == 'upload' def list_parts(self, start_part_number, max_part_count): start_part_number = start_part_number or 1 max_part_count = max_part_count or 100 parts = [ part.as_list_parts_dict() for part in self.parts if part is not None and start_part_number <= part.part_number ] if len(parts) <= max_part_count: next_part_number = None else: next_part_number = parts[max_part_count]['partNumber'] parts = parts[:max_part_count] return dict(parts=parts, nextPartNumber=next_part_number) def check_encryption(self, request_encryption: Optional[EncryptionSetting]): if not self.CHECK_ENCRYPTION: return file_mode, file_secret = self._get_encryption_mode_and_secret(self.server_side_encryption) request_mode, request_secret = self._get_encryption_mode_and_secret(request_encryption) if file_mode in (None, EncryptionMode.NONE): assert request_mode in (None, EncryptionMode.NONE) elif file_mode == EncryptionMode.SSE_B2: assert request_mode in (None, EncryptionMode.NONE, EncryptionMode.SSE_B2) elif file_mode == EncryptionMode.SSE_C: if request_mode != EncryptionMode.SSE_C or file_secret != request_secret: raise SSECKeyError() else: raise ValueError('Unsupported EncryptionMode: %s' % (file_mode)) def _get_encryption_mode_and_secret(self, encryption: Optional[EncryptionSetting]): if encryption is None: return None, None mode = encryption.mode if encryption.key is None: secret = None else: secret = encryption.key.secret return mode, secret FakeRequest = collections.namedtuple('FakeRequest', 'url headers') class FakeResponse: def __init__(self, account_auth_token_or_none, file_sim, url, range_=None): self.data_bytes = file_sim.data_bytes self.headers = file_sim.as_download_headers(account_auth_token_or_none, range_) self.url = url self.range_ = range_ if range_ is not None: self.data_bytes = self.data_bytes[range_[0]:range_[1] + 1] def iter_content(self, chunk_size=1): start = 0 rnd = random.Random(self.url) while start <= len(self.data_bytes): time.sleep(rnd.random() * 0.01) yield self.data_bytes[start:start + chunk_size] start += chunk_size @property def request(self): headers = CaseInsensitiveDict() if self.range_ is not None: headers['Range'] = '%s-%s' % self.range_ return FakeRequest(self.url, headers) def close(self): pass class BucketSimulator: # File IDs start at 9999 and count down, so they sort in the order # returned by list_file_versions. The IDs are strings. FIRST_FILE_NUMBER = 9999 FIRST_FILE_ID = str(FIRST_FILE_NUMBER) FILE_SIMULATOR_CLASS = FileSimulator RESPONSE_CLASS = FakeResponse MAX_SIMPLE_COPY_SIZE = 200 # should be same as RawSimulator.MIN_PART_SIZE def __init__( self, api, account_id, bucket_id, bucket_name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules=None, options_set=None, default_server_side_encryption=None, is_file_lock_enabled: Optional[bool] = None, replication: Optional[ReplicationConfiguration] = None, ): assert bucket_type in ['allPrivate', 'allPublic'] self.api = api self.account_id = account_id self.bucket_name = bucket_name self.bucket_id = bucket_id self.bucket_type = bucket_type self.bucket_info = bucket_info or {} self.cors_rules = cors_rules or [] self.lifecycle_rules = lifecycle_rules or [] self.options_set = options_set or set() self.revision = 1 self.upload_url_counter = iter(range(200)) # File IDs count down, so that the most recent will come first when they are sorted. self.file_id_counter = iter(range(self.FIRST_FILE_NUMBER, 0, -1)) self.upload_timestamp_counter = iter(range(5000, 9999)) self.file_id_to_file = dict() # It would be nice to use an OrderedDict for this, but 2.6 doesn't have it. self.file_name_and_id_to_file = dict() if default_server_side_encryption is None: default_server_side_encryption = EncryptionSetting(mode=EncryptionMode.NONE) self.default_server_side_encryption = default_server_side_encryption self.is_file_lock_enabled = is_file_lock_enabled self.default_retention = NO_RETENTION_BUCKET_SETTING self.replication = replication if self.replication is not None: assert self.replication.asReplicationSource is None or self.replication.asReplicationSource.rules assert self.replication.asReplicationDestination is None or self.replication.asReplicationDestination.sourceToDestinationKeyMapping def is_allowed_to_read_bucket_encryption_setting(self, account_auth_token): return self._check_capability(account_auth_token, 'readBucketEncryption') def is_allowed_to_read_bucket_retention(self, account_auth_token): return self._check_capability(account_auth_token, 'readBucketRetentions') def _check_capability(self, account_auth_token, capability): try: key = self.api.auth_token_to_key[account_auth_token] except KeyError: # looks like it's an upload token # fortunately BucketSimulator makes it easy to retrieve the true account_auth_token # from an upload url real_auth_token = account_auth_token.split('/')[-1] key = self.api.auth_token_to_key[real_auth_token] capabilities = key.get_allowed()['capabilities'] return capability in capabilities def bucket_dict(self, account_auth_token): default_sse = {'isClientAuthorizedToRead': False} if self.is_allowed_to_read_bucket_encryption_setting(account_auth_token): default_sse['isClientAuthorizedToRead'] = True default_sse['value'] = {'mode': self.default_server_side_encryption.mode.value} if self.default_server_side_encryption.algorithm is not None: default_sse['value']['algorithm' ] = self.default_server_side_encryption.algorithm.value else: default_sse['value'] = {'mode': EncryptionMode.UNKNOWN.value} if self.is_allowed_to_read_bucket_retention(account_auth_token): file_lock_configuration = { 'isClientAuthorizedToRead': True, 'value': { 'defaultRetention': { 'mode': self.default_retention.mode.value, 'period': self.default_retention.period.as_dict() if self.default_retention.period else None, }, 'isFileLockEnabled': self.is_file_lock_enabled, }, } # yapf: disable else: file_lock_configuration = {'isClientAuthorizedToRead': False, 'value': None} replication = self.replication and { 'isClientAuthorizedToRead': True, 'value': self.replication.as_dict(), } return dict( accountId=self.account_id, bucketName=self.bucket_name, bucketId=self.bucket_id, bucketType=self.bucket_type, bucketInfo=self.bucket_info, corsRules=self.cors_rules, lifecycleRules=self.lifecycle_rules, options=self.options_set, revision=self.revision, defaultServerSideEncryption=default_sse, fileLockConfiguration=file_lock_configuration, replicationConfiguration=replication, ) def cancel_large_file(self, file_id): file_sim = self.file_id_to_file[file_id] key = (file_sim.name, file_id) del self.file_name_and_id_to_file[key] del self.file_id_to_file[file_id] return dict( accountId=self.account_id, bucketId=self.bucket_id, fileId=file_id, fileName=file_sim.name ) # yapf: disable def delete_file_version(self, file_id, file_name): key = (file_name, file_id) file_sim = self.file_name_and_id_to_file[key] del self.file_name_and_id_to_file[key] del self.file_id_to_file[file_id] return dict(fileId=file_id, fileName=file_name, uploadTimestamp=file_sim.upload_timestamp) def download_file_by_id( self, account_auth_token_or_none, file_id, url, range_=None, encryption: Optional[EncryptionSetting] = None, ): file_sim = self.file_id_to_file[file_id] file_sim.check_encryption(encryption) return self._download_file_sim(account_auth_token_or_none, file_sim, url, range_=range_) def download_file_by_name( self, account_auth_token_or_none, file_name, url, range_=None, encryption: Optional[EncryptionSetting] = None, ): files = self.list_file_names(self.api.current_token, file_name, 1)['files'] # token is not important here if len(files) == 0: raise FileNotPresent(file_id_or_name=file_name) file_dict = files[0] if file_dict['fileName'] != file_name: raise FileNotPresent(file_id_or_name=file_name) file_sim = self.file_name_and_id_to_file[(file_name, file_dict['fileId'])] if not file_sim.is_visible(): raise FileNotPresent(file_id_or_name=file_name) file_sim.check_encryption(encryption) return self._download_file_sim(account_auth_token_or_none, file_sim, url, range_=range_) def _download_file_sim(self, account_auth_token_or_none, file_sim, url, range_=None): return ResponseContextManager( self.RESPONSE_CLASS( account_auth_token_or_none, file_sim, url, range_, ) ) def finish_large_file(self, account_auth_token, file_id, part_sha1_array): file_sim = self.file_id_to_file[file_id] file_sim.finish(part_sha1_array) return file_sim.as_upload_result(account_auth_token) def get_file_info_by_id(self, account_auth_token, file_id): return self.file_id_to_file[file_id].as_upload_result(account_auth_token) def get_file_info_by_name(self, account_auth_token, file_name): for ((name, id), file) in self.file_name_and_id_to_file.items(): if file_name == name: return file.as_download_headers(account_auth_token_or_none=account_auth_token) raise FileNotPresent(file_id_or_name=file_name, bucket_name=self.bucket_name) def get_upload_url(self, account_auth_token): upload_id = next(self.upload_url_counter) upload_url = 'https://upload.example.com/%s/%d/%s' % ( self.bucket_id, upload_id, account_auth_token ) return dict(bucketId=self.bucket_id, uploadUrl=upload_url, authorizationToken=upload_url) def get_upload_part_url(self, account_auth_token, file_id): upload_url = 'https://upload.example.com/part/%s/%d/%s' % ( file_id, random.randint(1, 10**9), account_auth_token ) return dict(bucketId=self.bucket_id, uploadUrl=upload_url, authorizationToken=upload_url) def hide_file(self, account_auth_token, file_name): file_id = self._next_file_id() file_sim = self.FILE_SIMULATOR_CLASS( self.account_id, self, file_id, 'hide', file_name, None, "none", {}, b'', next(self.upload_timestamp_counter) ) self.file_id_to_file[file_id] = file_sim self.file_name_and_id_to_file[file_sim.sort_key()] = file_sim return file_sim.as_list_files_dict(account_auth_token) def update_file_retention( self, account_auth_token, file_id, file_name, file_retention: FileRetentionSetting, bypass_governance: bool = False, ): file_sim = self.file_id_to_file[file_id] assert self.is_file_lock_enabled assert file_sim.name == file_name # TODO: check bypass etc file_sim.file_retention = file_retention return { 'fileId': file_id, 'fileName': file_name, 'fileRetention': file_sim.file_retention.serialize_to_json_for_request(), } def update_file_legal_hold( self, account_auth_token, file_id, file_name, legal_hold: LegalHold, ): file_sim = self.file_id_to_file[file_id] assert self.is_file_lock_enabled assert file_sim.name == file_name file_sim.legal_hold = legal_hold return { 'fileId': file_id, 'fileName': file_name, 'legalHold': legal_hold.to_server(), } def copy_file( self, account_auth_token, file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_bucket_id=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): if metadata_directive is not None: assert metadata_directive in tuple(MetadataDirectiveMode), metadata_directive if metadata_directive is MetadataDirectiveMode.COPY and ( content_type is not None or file_info is not None ): raise InvalidMetadataDirective( 'content_type and file_info should be None when metadata_directive is COPY' ) elif metadata_directive is MetadataDirectiveMode.REPLACE and content_type is None: raise InvalidMetadataDirective( 'content_type cannot be None when metadata_directive is REPLACE' ) file_sim = self.file_id_to_file[file_id] file_sim.check_encryption(source_server_side_encryption) new_file_id = self._next_file_id() data_bytes = get_bytes_range(file_sim.data_bytes, bytes_range) if len(data_bytes) > self.MAX_SIMPLE_COPY_SIZE: raise CopySourceTooBig( 'Copy source too big: %i' % (len(data_bytes),), 'bad_request', len(data_bytes), ) destination_bucket = self.api.bucket_id_to_bucket.get(destination_bucket_id, self) sse = destination_server_side_encryption or self.default_server_side_encryption copy_file_sim = self.FILE_SIMULATOR_CLASS( self.account_id, destination_bucket, new_file_id, 'upload', new_file_name, file_sim.content_type, hex_sha1_of_bytes(data_bytes), # we hash here again because bytes_range may not cover the full source file_sim.file_info, data_bytes, next(self.upload_timestamp_counter), server_side_encryption=sse, file_retention=file_retention, legal_hold=legal_hold, ) # yapf: disable destination_bucket.file_id_to_file[copy_file_sim.file_id] = copy_file_sim destination_bucket.file_name_and_id_to_file[copy_file_sim.sort_key()] = copy_file_sim if metadata_directive is MetadataDirectiveMode.REPLACE: copy_file_sim.content_type = content_type copy_file_sim.file_info = file_info or file_sim.file_info ## long term storage of that file has action="upload", but here we need to return action="copy", just this once #class TestFileVersionFactory(FileVersionFactory): # FILE_VERSION_CLASS = self.FILE_SIMULATOR_CLASS #file_version_dict = copy_file_sim.as_upload_result(account_auth_token) #del file_version_dict['action'] #print(file_version_dict) #copy_file_sim_with_action_copy = TestFileVersionFactory(self.api).from_api_response(file_version_dict, force_action='copy') #return copy_file_sim_with_action_copy # TODO: the code above cannot be used right now because FileSimulator.__init__ is incompatible with FileVersionFactory / FileVersion.__init__ - refactor is needed # for now we'll just return the newly constructed object with a copy action... return self.FILE_SIMULATOR_CLASS( self.account_id, destination_bucket, new_file_id, 'copy', new_file_name, copy_file_sim.content_type, copy_file_sim.content_sha1, copy_file_sim.file_info, data_bytes, copy_file_sim.upload_timestamp, server_side_encryption=sse, file_retention=file_retention, legal_hold=legal_hold, ) def list_file_names( self, account_auth_token, start_file_name=None, max_file_count=None, prefix=None, ): assert prefix is None or start_file_name is None or start_file_name.startswith(prefix ), locals() start_file_name = start_file_name or '' max_file_count = max_file_count or 100 result_files = [] next_file_name = None prev_file_name = None for key in sorted(self.file_name_and_id_to_file): (file_name, file_id) = key assert file_id if start_file_name <= file_name and file_name != prev_file_name: if prefix is not None and not file_name.startswith(prefix): break prev_file_name = file_name file_sim = self.file_name_and_id_to_file[key] if file_sim.is_visible(): result_files.append(file_sim.as_list_files_dict(account_auth_token)) if len(result_files) == max_file_count: next_file_name = file_sim.name + ' ' break else: logger.debug('skipping invisible file during listing: %s', key) return dict(files=result_files, nextFileName=next_file_name) def list_file_versions( self, account_auth_token, start_file_name=None, start_file_id=None, max_file_count=None, prefix=None, ): assert prefix is None or start_file_name is None or start_file_name.startswith(prefix ), locals() start_file_name = start_file_name or '' start_file_id = start_file_id or '' max_file_count = max_file_count or 100 result_files = [] next_file_name = None next_file_id = None for key in sorted(self.file_name_and_id_to_file): (file_name, file_id) = key if (start_file_name < file_name) or ( start_file_name == file_name and (start_file_id == '' or int(start_file_id) <= int(file_id)) ): file_sim = self.file_name_and_id_to_file[key] if prefix is not None and not file_name.startswith(prefix): break result_files.append(file_sim.as_list_files_dict(account_auth_token)) if len(result_files) == max_file_count: next_file_name = file_sim.name next_file_id = str(int(file_id) + 1) break return dict(files=result_files, nextFileName=next_file_name, nextFileId=next_file_id) def list_parts(self, file_id, start_part_number, max_part_count): file_sim = self.file_id_to_file[file_id] return file_sim.list_parts(start_part_number, max_part_count) def list_unfinished_large_files( self, account_auth_token, start_file_id=None, max_file_count=None, prefix=None ): start_file_id = start_file_id or self.FIRST_FILE_ID max_file_count = max_file_count or 100 all_unfinished_ids = set( k for (k, v) in self.file_id_to_file.items() if v.action == 'start' and k <= start_file_id and (prefix is None or v.name.startswith(prefix)) ) ids_in_order = sorted(all_unfinished_ids, reverse=True) file_dict_list = [ file_sim.as_start_large_file_result(account_auth_token) for file_sim in ( self.file_id_to_file[file_id] for file_id in ids_in_order[:max_file_count] ) ] # yapf: disable next_file_id = None if len(file_dict_list) == max_file_count: next_file_id = str(int(file_dict_list[-1]['fileId']) - 1) return dict(files=file_dict_list, nextFileId=next_file_id) def start_large_file( self, account_auth_token, file_name, content_type, file_info, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): file_id = self._next_file_id() sse = server_side_encryption or self.default_server_side_encryption if sse: # FIXME: remove this part when RawApi<->Encryption adapters are implemented properly file_info = sse.add_key_id_to_file_info(file_info) file_sim = self.FILE_SIMULATOR_CLASS( self.account_id, self, file_id, 'start', file_name, content_type, 'none', file_info, None, next(self.upload_timestamp_counter), server_side_encryption=sse, file_retention=file_retention, legal_hold=legal_hold, ) # yapf: disable self.file_id_to_file[file_id] = file_sim self.file_name_and_id_to_file[file_sim.sort_key()] = file_sim return file_sim.as_start_large_file_result(account_auth_token) def _update_bucket( self, bucket_type=None, bucket_info=None, cors_rules=None, lifecycle_rules=None, if_revision_is: Optional[int] = None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, replication: Optional[ReplicationConfiguration] = None, ): if if_revision_is is not None and self.revision != if_revision_is: raise Conflict() if bucket_type is not None: self.bucket_type = bucket_type if bucket_info is not None: self.bucket_info = bucket_info if cors_rules is not None: self.cors_rules = cors_rules if lifecycle_rules is not None: self.lifecycle_rules = lifecycle_rules if default_server_side_encryption is not None: self.default_server_side_encryption = default_server_side_encryption if default_retention: self.default_retention = default_retention self.revision += 1 self.replication = replication return self.bucket_dict(self.api.current_token) def upload_file( self, upload_id: str, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_infos: dict, data_stream, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): data_bytes = self._simulate_chunked_post(data_stream, content_length) assert len(data_bytes) == content_length if content_sha1 == HEX_DIGITS_AT_END: content_sha1 = data_bytes[-40:].decode() data_bytes = data_bytes[0:-40] content_length -= 40 elif len(content_sha1) != 40: raise ValueError(content_sha1) computed_sha1 = hex_sha1_of_bytes(data_bytes) if content_sha1 != computed_sha1: raise FileSha1Mismatch(file_name) if content_sha1 == 'do_not_verify': content_sha1 = UNVERIFIED_CHECKSUM_PREFIX + computed_sha1 file_id = self._next_file_id() encryption = server_side_encryption or self.default_server_side_encryption if encryption: # FIXME: remove this part when RawApi<->Encryption adapters are implemented properly file_infos = encryption.add_key_id_to_file_info(file_infos) file_sim = self.FILE_SIMULATOR_CLASS( self.account_id, self, file_id, 'upload', file_name, content_type, content_sha1, file_infos, data_bytes, next(self.upload_timestamp_counter), server_side_encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) self.file_id_to_file[file_id] = file_sim self.file_name_and_id_to_file[file_sim.sort_key()] = file_sim return file_sim.as_upload_result(upload_auth_token) def upload_part( self, file_id, part_number, content_length, sha1_sum, input_stream, server_side_encryption: Optional[EncryptionSetting] = None, ): file_sim = self.file_id_to_file[file_id] part_data = self._simulate_chunked_post(input_stream, content_length) assert len(part_data) == content_length if sha1_sum == HEX_DIGITS_AT_END: sha1_sum = part_data[-40:].decode() part_data = part_data[0:-40] content_length -= 40 computed_sha1 = hex_sha1_of_bytes(part_data) if sha1_sum != computed_sha1: raise PartSha1Mismatch(file_id) part = PartSimulator(file_sim.file_id, part_number, content_length, sha1_sum, part_data) file_sim.add_part(part_number, part) result = dict( fileId=file_id, partNumber=part_number, contentLength=content_length, contentSha1=sha1_sum, ) # yapf: disable if server_side_encryption is not None: result['serverSideEncryption'] = server_side_encryption.serialize_to_json_for_request() return result def _simulate_chunked_post( self, stream, content_length, min_chunks=4, max_chunk_size=8096, simulate_retry=True ): chunk_size = max_chunk_size chunks_num = self._chunks_number(content_length, chunk_size) if chunks_num < min_chunks: chunk_size = max(content_length // min_chunks, 1) loop_count = 2 if simulate_retry else 1 stream_data = None for _ in range(loop_count): chunks = [] stream.seek(0) # we always do this in `do_post` in `b2http` so we want it here *always* while True: data = stream.read(chunk_size) chunks.append(data) if not data: break _stream_data = b''.join(chunks) if stream_data is not None: assert _stream_data == stream_data stream_data = _stream_data return stream_data def _chunks_number(self, content_length, chunk_size): chunks_number = content_length // chunk_size if content_length % chunk_size > 0: chunks_number = chunks_number + 1 return chunks_number def _next_file_id(self): return str(next(self.file_id_counter)) class RawSimulator(AbstractRawApi): """ Implement the same interface as B2RawHTTPApi by simulating all of the calls and keeping state in memory. The intended use for this class is for unit tests that test things built on top of B2RawHTTPApi. """ BUCKET_SIMULATOR_CLASS = BucketSimulator API_URL = 'http://api.example.com' S3_API_URL = 'http://s3.api.example.com' DOWNLOAD_URL = 'http://download.example.com' MIN_PART_SIZE = 200 MAX_PART_ID = 10000 # This is the maximum duration in seconds that an application key can be valid (1000 days). MAX_DURATION_IN_SECONDS = 86400000 UPLOAD_PART_MATCHER = re.compile('https://upload.example.com/part/([^/]*)') UPLOAD_URL_MATCHER = re.compile(r'https://upload.example.com/([^/]*)/([^/]*)') DOWNLOAD_URL_MATCHER = re.compile( DOWNLOAD_URL + '(?:' + '|'.join( ( '/b2api/v[0-9]+/b2_download_file_by_id\?fileId=(?P[^/]+)', '/file/(?P[^/]+)/(?P.+)', ) ) + ')$' ) # yapf: disable def __init__(self, b2_http=None): # Map from application_key_id to KeySimulator. # The entry for the master application key ID is for the master application # key for the account, and the entries with non-master application keys # are for keys created b2 createKey(). self.key_id_to_key = dict() # Map from auth token to the KeySimulator for it. self.auth_token_to_key = dict() # Set of auth tokens that have expired self.expired_auth_tokens = set() # Map from auth token to a lock that upload procedure acquires # when utilizing the token self.currently_used_auth_tokens = collections.defaultdict(threading.Lock) # Counter for generating auth tokens. self.auth_token_counter = 0 # Counter for generating account IDs an their matching master application keys. self.account_counter = 0 self.bucket_name_to_bucket = dict() self.bucket_id_to_bucket = dict() self.bucket_id_counter = iter(range(100)) self.file_id_to_bucket_id = {} self.all_application_keys = [] self.app_key_counter = 0 self.upload_errors = [] def expire_auth_token(self, auth_token): """ Simulate the auth token expiring. The next call that tries to use this auth token will get an auth_token_expired error. """ assert auth_token in self.auth_token_to_key self.expired_auth_tokens.add(auth_token) def create_account(self): """ Return (accountId, masterApplicationKey) for a newly created account. """ # Pick the IDs for the account and the key account_id = 'account-%d' % (self.account_counter,) master_key = 'masterKey-%d' % (self.account_counter,) self.account_counter += 1 # Create the key self.key_id_to_key[account_id] = KeySimulator( account_id=account_id, name='master', application_key_id=account_id, key=master_key, capabilities=ALL_CAPABILITIES, expiration_timestamp_or_none=None, bucket_id_or_none=None, bucket_name_or_none=None, name_prefix_or_none=None, ) # Return the info return (account_id, master_key) def set_upload_errors(self, errors): """ Store a sequence of exceptions to raise on upload. Each one will be raised in turn, until they are all gone. Then the next upload will succeed. """ assert len(self.upload_errors) == 0 self.upload_errors = errors def authorize_account(self, realm_url, application_key_id, application_key): key_sim = self.key_id_to_key.get(application_key_id) if key_sim is None: raise InvalidAuthToken('application key ID not valid', 'unauthorized') if application_key != key_sim.key: raise InvalidAuthToken('secret key is wrong', 'unauthorized') auth_token = 'auth_token_%d' % (self.auth_token_counter,) self.current_token = auth_token self.auth_token_counter += 1 self.auth_token_to_key[auth_token] = key_sim allowed = key_sim.get_allowed() bucketId = allowed.get('bucketId') if (bucketId is not None) and (bucketId in self.bucket_id_to_bucket): allowed['bucketName'] = self.bucket_id_to_bucket[bucketId].bucket_name else: allowed['bucketName'] = None return dict( accountId=key_sim.account_id, authorizationToken=auth_token, apiUrl=self.API_URL, downloadUrl=self.DOWNLOAD_URL, recommendedPartSize=self.MIN_PART_SIZE, absoluteMinimumPartSize=self.MIN_PART_SIZE, allowed=allowed, s3ApiUrl=self.S3_API_URL, ) def cancel_large_file(self, api_url, account_auth_token, file_id): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') return bucket.cancel_large_file(file_id) def create_bucket( self, api_url, account_auth_token, account_id, bucket_name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, is_file_lock_enabled: Optional[bool] = None, replication: Optional[ReplicationConfiguration] = None, ): if not re.match(r'^[-a-zA-Z0-9]*$', bucket_name): raise BadJson('illegal bucket name: ' + bucket_name) self._assert_account_auth(api_url, account_auth_token, account_id, 'writeBuckets') if bucket_name in self.bucket_name_to_bucket: raise DuplicateBucketName(bucket_name) bucket_id = 'bucket_' + str(next(self.bucket_id_counter)) bucket = self.BUCKET_SIMULATOR_CLASS( self, account_id, bucket_id, bucket_name, bucket_type, bucket_info, cors_rules, lifecycle_rules, # watch out for options! default_server_side_encryption=default_server_side_encryption, is_file_lock_enabled=is_file_lock_enabled, replication=replication, ) self.bucket_name_to_bucket[bucket_name] = bucket self.bucket_id_to_bucket[bucket_id] = bucket return bucket.bucket_dict(account_auth_token) # TODO it should be an object, right? def create_key( self, api_url, account_auth_token, account_id, capabilities, key_name, valid_duration_seconds, bucket_id, name_prefix, ): if not re.match(r'^[A-Za-z0-9-]{1,100}$', key_name): raise BadJson('illegal key name: ' + key_name) if valid_duration_seconds is not None: if valid_duration_seconds < 1 or valid_duration_seconds > self.MAX_DURATION_IN_SECONDS: raise BadJson( 'valid duration must be greater than 0, and less than 1000 days in seconds' ) self._assert_account_auth(api_url, account_auth_token, account_id, 'writeKeys') if valid_duration_seconds is None: expiration_timestamp_or_none = None else: expiration_timestamp_or_none = int(time.time() + valid_duration_seconds) index = self.app_key_counter self.app_key_counter += 1 application_key_id = 'appKeyId%d' % (index,) app_key = 'appKey%d' % (index,) if bucket_id is None: bucket_name_or_none = None else: bucket_name_or_none = self._get_bucket_by_id(bucket_id).bucket_name key_sim = KeySimulator( account_id=account_id, name=key_name, application_key_id=application_key_id, key=app_key, capabilities=capabilities, expiration_timestamp_or_none=expiration_timestamp_or_none, bucket_id_or_none=bucket_id, bucket_name_or_none=bucket_name_or_none, name_prefix_or_none=name_prefix, ) self.key_id_to_key[application_key_id] = key_sim self.all_application_keys.append(key_sim) return key_sim.as_created_key() def delete_file_version(self, api_url, account_auth_token, file_id, file_name): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'deleteFiles') return bucket.delete_file_version(file_id, file_name) def update_file_retention( self, api_url, account_auth_token, file_id, file_name, file_retention: FileRetentionSetting, bypass_governance: bool = False, ): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) return bucket.update_file_retention( account_auth_token, file_id, file_name, file_retention, bypass_governance, ) def update_file_legal_hold( self, api_url, account_auth_token, file_id, file_name, legal_hold: bool, ): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) return bucket.update_file_legal_hold( account_auth_token, file_id, file_name, legal_hold, ) def delete_bucket(self, api_url, account_auth_token, account_id, bucket_id): self._assert_account_auth(api_url, account_auth_token, account_id, 'deleteBuckets') bucket = self._get_bucket_by_id(bucket_id) del self.bucket_name_to_bucket[bucket.bucket_name] del self.bucket_id_to_bucket[bucket_id] return bucket.bucket_dict(account_auth_token) def download_file_from_url( self, account_auth_token_or_none, url, range_=None, encryption: Optional[EncryptionSetting] = None ): # TODO: check auth token if bucket is not public matcher = self.DOWNLOAD_URL_MATCHER.match(url) assert matcher is not None, url groupdict = matcher.groupdict() file_id = groupdict['file_id'] bucket_name = groupdict['bucket_name'] file_name = groupdict['file_name'] if file_id is not None: bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) return bucket.download_file_by_id( account_auth_token_or_none, file_id, range_=range_, url=url, encryption=encryption, ) elif bucket_name is not None and file_name is not None: bucket = self._get_bucket_by_name(bucket_name) return bucket.download_file_by_name( account_auth_token_or_none, b2_url_decode(file_name), range_=range_, url=url, encryption=encryption, ) else: assert False def delete_key(self, api_url, account_auth_token, application_key_id): assert api_url == self.API_URL key_sim = self.key_id_to_key.pop(application_key_id, None) if key_sim is None: raise BadRequest( 'application key does not exist: %s' % (application_key_id,), 'bad_request', ) return key_sim.as_key() def finish_large_file(self, api_url, account_auth_token, file_id, part_sha1_array): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') return bucket.finish_large_file(account_auth_token, file_id, part_sha1_array) def get_download_authorization( self, api_url, account_auth_token, bucket_id, file_name_prefix, valid_duration_in_seconds ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'shareFiles') return { 'bucketId': bucket_id, 'fileNamePrefix': file_name_prefix, 'authorizationToken': 'fake_download_auth_token_%s_%s_%d' % ( bucket_id, b2_url_encode(file_name_prefix), valid_duration_in_seconds, ) } def get_file_info_by_id(self, api_url, account_auth_token, file_id): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) return bucket.get_file_info_by_id(account_auth_token, file_id) def get_file_info_by_name(self, api_url, account_auth_token, bucket_name, file_name): bucket = self._get_bucket_by_name(bucket_name) info = bucket.get_file_info_by_name(account_auth_token, file_name) return info def get_upload_url(self, api_url, account_auth_token, bucket_id): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') return self._get_bucket_by_id(bucket_id).get_upload_url(account_auth_token) def get_upload_part_url(self, api_url, account_auth_token, file_id): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') return self._get_bucket_by_id(bucket_id).get_upload_part_url(account_auth_token, file_id) def hide_file(self, api_url, account_auth_token, bucket_id, file_name): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') response = bucket.hide_file(account_auth_token, file_name) self.file_id_to_bucket_id[response['fileId']] = bucket_id return response def copy_file( self, api_url, account_auth_token, source_file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_bucket_id=None, destination_server_side_encryption=None, source_server_side_encryption=None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): bucket_id = self.file_id_to_bucket_id[source_file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') if destination_bucket_id: # TODO: Handle and raise proper exception after server docs get updated dest_bucket = self.bucket_id_to_bucket[destination_bucket_id] assert dest_bucket.account_id == bucket.account_id else: dest_bucket = bucket copy_file_sim = bucket.copy_file( account_auth_token, source_file_id, new_file_name, bytes_range, metadata_directive, content_type, file_info, destination_bucket_id, destination_server_side_encryption=destination_server_side_encryption, source_server_side_encryption=source_server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, ) return copy_file_sim.as_upload_result(account_auth_token) def copy_part( self, api_url, account_auth_token, source_file_id, large_file_id, part_number, bytes_range=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, ): if destination_server_side_encryption is not None and destination_server_side_encryption.mode == EncryptionMode.SSE_B2: raise ValueError( 'unsupported sse mode for copy_part!' ) # SSE-B2 is only to be marked in b2_start_large_file src_bucket_id = self.file_id_to_bucket_id[source_file_id] src_bucket = self._get_bucket_by_id(src_bucket_id) dest_bucket_id = self.file_id_to_bucket_id[large_file_id] dest_bucket = self._get_bucket_by_id(dest_bucket_id) self._assert_account_auth(api_url, account_auth_token, dest_bucket.account_id, 'writeFiles') file_sim = src_bucket.file_id_to_file[source_file_id] file_sim.check_encryption(source_server_side_encryption) data_bytes = get_bytes_range(file_sim.data_bytes, bytes_range) data_stream = StreamWithHash(io.BytesIO(data_bytes), len(data_bytes)) content_length = len(data_stream) sha1_sum = HEX_DIGITS_AT_END return dest_bucket.upload_part( large_file_id, part_number, content_length, sha1_sum, data_stream, server_side_encryption=destination_server_side_encryption, ) def list_buckets( self, api_url, account_auth_token, account_id, bucket_id=None, bucket_name=None ): # First, map the bucket name to a bucket_id, so that we can check auth. if bucket_name is None: bucket_id_for_auth = bucket_id else: bucket_id_for_auth = self._get_bucket_id_or_none_for_bucket_name(bucket_name) self._assert_account_auth( api_url, account_auth_token, account_id, 'listBuckets', bucket_id_for_auth ) # Do the query sorted_buckets = [ self.bucket_name_to_bucket[name] for name in sorted(self.bucket_name_to_bucket) ] bucket_list = [ bucket.bucket_dict(account_auth_token) for bucket in sorted_buckets if self._bucket_matches(bucket, bucket_id, bucket_name) ] return dict(buckets=bucket_list) def _get_bucket_id_or_none_for_bucket_name(self, bucket_name): for bucket in self.bucket_name_to_bucket.values(): if bucket.bucket_name == bucket_name: return bucket.bucket_id def _bucket_matches(self, bucket, bucket_id, bucket_name): return ( (bucket_id is None or bucket.bucket_id == bucket_id) and (bucket_name is None or bucket.bucket_name == bucket_name) ) def list_file_names( self, api_url, account_auth_token, bucket_id, start_file_name=None, max_file_count=None, prefix=None, ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth( api_url, account_auth_token, bucket.account_id, 'listFiles', bucket_id=bucket_id, file_name=prefix, ) return bucket.list_file_names(account_auth_token, start_file_name, max_file_count, prefix) def list_file_versions( self, api_url, account_auth_token, bucket_id, start_file_name=None, start_file_id=None, max_file_count=None, prefix=None, ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth( api_url, account_auth_token, bucket.account_id, 'listFiles', bucket_id=bucket_id, file_name=prefix, ) return bucket.list_file_versions( account_auth_token, start_file_name, start_file_id, max_file_count, prefix, ) def list_keys( self, api_url, account_auth_token, account_id, max_key_count=1000, start_application_key_id=None ): self._assert_account_auth(api_url, account_auth_token, account_id, 'listKeys') next_application_key_id = None all_keys_sorted = sorted(self.all_application_keys, key=lambda key: key.application_key_id) if start_application_key_id is None: keys = all_keys_sorted[:max_key_count] if max_key_count < len(all_keys_sorted): next_application_key_id = all_keys_sorted[max_key_count].application_key_id else: keys = [] got_already = 0 for ind, key in enumerate(all_keys_sorted): if key.application_key_id >= start_application_key_id: keys.append(key) got_already += 1 if got_already == max_key_count: if ind < len(all_keys_sorted) - 1: next_application_key_id = all_keys_sorted[ind + 1].application_key_id break key_dicts = map(lambda key: key.as_key(), keys) return dict(keys=list(key_dicts), nextApplicationKeyId=next_application_key_id) def list_parts(self, api_url, account_auth_token, file_id, start_part_number, max_part_count): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') return bucket.list_parts(file_id, start_part_number, max_part_count) def list_unfinished_large_files( self, api_url, account_auth_token, bucket_id, start_file_id=None, max_file_count=None, prefix=None ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth( api_url, account_auth_token, bucket.account_id, 'listFiles', file_name=prefix ) start_file_id = start_file_id or '' max_file_count = max_file_count or 100 return bucket.list_unfinished_large_files( account_auth_token, start_file_id, max_file_count, prefix ) def start_large_file( self, api_url, account_auth_token, bucket_id, file_name, content_type, file_info, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') result = bucket.start_large_file( account_auth_token, file_name, content_type, file_info, server_side_encryption, file_retention, legal_hold, ) self.file_id_to_bucket_id[result['fileId']] = bucket_id return result def update_bucket( self, api_url, account_auth_token, account_id, bucket_id, bucket_type=None, bucket_info=None, cors_rules=None, lifecycle_rules=None, if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, replication: Optional[ReplicationConfiguration] = None, ): assert bucket_type or bucket_info or cors_rules or lifecycle_rules or default_server_side_encryption or replication bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeBuckets') return bucket._update_bucket( bucket_type=bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, replication=replication, ) @classmethod def get_upload_file_headers( cls, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_infos: dict, server_side_encryption: Optional[EncryptionSetting], file_retention: Optional[FileRetentionSetting], legal_hold: Optional[LegalHold], ) -> dict: # fix to allow calculating headers on unknown key - only for simulation if server_side_encryption is not None \ and server_side_encryption.mode == EncryptionMode.SSE_C \ and server_side_encryption.key.secret is None: server_side_encryption.key.secret = b'secret' return super().get_upload_file_headers( upload_auth_token=upload_auth_token, file_name=file_name, content_length=content_length, content_type=content_type, content_sha1=content_sha1, file_infos=file_infos, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, ) def upload_file( self, upload_url: str, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_infos: dict, data_stream, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): with ConcurrentUsedAuthTokenGuard( self.currently_used_auth_tokens[upload_auth_token], upload_auth_token ): assert upload_url == upload_auth_token url_match = self.UPLOAD_URL_MATCHER.match(upload_url) if url_match is None: raise BadUploadUrl(upload_url) if self.upload_errors: raise self.upload_errors.pop(0) bucket_id, upload_id = url_match.groups() bucket = self._get_bucket_by_id(bucket_id) if server_side_encryption is not None: assert server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C ) file_infos = server_side_encryption.add_key_id_to_file_info(file_infos) # we don't really need headers further on # but we still simulate their calculation _ = self.get_upload_file_headers( upload_auth_token=upload_auth_token, file_name=file_name, content_length=content_length, content_type=content_type, content_sha1=content_sha1, file_infos=file_infos, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, ) response = bucket.upload_file( upload_id, upload_auth_token, file_name, content_length, content_type, content_sha1, file_infos, data_stream, server_side_encryption, file_retention, legal_hold, ) file_id = response['fileId'] self.file_id_to_bucket_id[file_id] = bucket_id return response def upload_part( self, upload_url, upload_auth_token, part_number, content_length, sha1_sum, input_stream, server_side_encryption: Optional[EncryptionSetting] = None, ): with ConcurrentUsedAuthTokenGuard( self.currently_used_auth_tokens[upload_auth_token], upload_auth_token ): url_match = self.UPLOAD_PART_MATCHER.match(upload_url) if url_match is None: raise BadUploadUrl(upload_url) elif part_number > self.MAX_PART_ID: raise BadRequest('Part number must be in range 1 - 10000', 'bad_request') file_id = url_match.group(1) bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) part = bucket.upload_part( file_id, part_number, content_length, sha1_sum, input_stream, server_side_encryption ) return part def _assert_account_auth( self, api_url, account_auth_token, account_id, capability, bucket_id=None, file_name=None ): key_sim = self.auth_token_to_key.get(account_auth_token) assert key_sim is not None assert api_url == self.API_URL assert account_id == key_sim.account_id if account_auth_token in self.expired_auth_tokens: raise InvalidAuthToken('auth token expired', 'auth_token_expired') if capability not in key_sim.capabilities: raise Unauthorized('', 'unauthorized') if key_sim.bucket_id_or_none is not None and key_sim.bucket_id_or_none != bucket_id: raise Unauthorized('', 'unauthorized') if key_sim.name_prefix_or_none is not None: if file_name is not None and not file_name.startswith(key_sim.name_prefix_or_none): raise Unauthorized('', 'unauthorized') def _get_bucket_by_id(self, bucket_id): if bucket_id not in self.bucket_id_to_bucket: raise NonExistentBucket(bucket_id) return self.bucket_id_to_bucket[bucket_id] def _get_bucket_by_name(self, bucket_name): if bucket_name not in self.bucket_name_to_bucket: raise NonExistentBucket(bucket_name) return self.bucket_name_to_bucket[bucket_name] b2-sdk-python-1.17.3/b2sdk/replication/000077500000000000000000000000001426424117700175325ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/replication/__init__.py000066400000000000000000000004501426424117700216420ustar00rootroot00000000000000###################################################################### # # File: b2sdk/replication/__init__.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/replication/monitoring.py000066400000000000000000000210271426424117700222730ustar00rootroot00000000000000###################################################################### # # File: b2sdk/replication/monitoring.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import sys from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from queue import Queue from typing import ClassVar, Iterator, Optional, Tuple, Type from ..api import B2Api from ..bucket import Bucket from ..encryption.setting import EncryptionMode from ..file_lock import NO_RETENTION_FILE_SETTING, LegalHold from ..scan.folder import B2Folder from ..scan.path import B2Path from ..scan.policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from ..scan.report import ProgressReport from ..scan.scan import AbstractScanReport, AbstractScanResult, CountAndSampleScanReport, zip_folders from .setting import ReplicationRule from .types import ReplicationStatus @dataclass(frozen=True) class ReplicationScanResult(AbstractScanResult): """ Some attributes of source and destination files and their relations which are meaningful for replication monitoring and troubleshooting. Please be aware that only latest file versions are inspected, so any previous file versions are not represented in these results. """ # source attrs source_replication_status: Optional[ReplicationStatus] = None source_has_hide_marker: Optional[bool] = None source_has_sse_c_enabled: Optional[bool] = None source_has_large_metadata: Optional[bool] = None source_has_file_retention: Optional[bool] = None source_has_legal_hold: Optional[bool] = None # destination attrs destination_replication_status: Optional[ReplicationStatus] = None # source & destination relation attrs metadata_differs: Optional[bool] = None hash_differs: Optional[bool] = None LARGE_METADATA_SIZE: ClassVar[int] = 2048 @classmethod def from_files( cls, source_file: Optional[B2Path] = None, destination_file: Optional[B2Path] = None ) -> 'ReplicationScanResult': params = {} if source_file: source_file_version = source_file.selected_version params.update( { 'source_replication_status': source_file_version.replication_status, 'source_has_hide_marker': not source_file.is_visible(), 'source_has_sse_c_enabled': source_file_version.server_side_encryption.mode == EncryptionMode.SSE_C, 'source_has_large_metadata': source_file_version.has_large_header, 'source_has_file_retention': source_file_version.file_retention is not NO_RETENTION_FILE_SETTING, 'source_has_legal_hold': source_file_version.legal_hold is LegalHold.ON, } ) if destination_file: params.update( { 'destination_replication_status': destination_file.selected_version.replication_status, } ) if source_file and destination_file: source_version = source_file.selected_version destination_version = destination_file.selected_version params.update( { 'metadata_differs': source_version.file_info != destination_version.file_info, 'hash_differs': (source_version.content_md5 != destination_version.content_md5) or (source_version.content_sha1 != destination_version.content_sha1) } ) return cls(**params) @dataclass class ReplicationReport(CountAndSampleScanReport): SCAN_RESULT_CLASS = ReplicationScanResult @dataclass class ReplicationMonitor: """ Calculates source and (optionally) destination replication statistics. :param b2sdk.v2.Bucket bucket: replication source bucket :param b2sdk.v2.ReplicationRule rule: replication rule to be monitored; should belong to `bucket`'s replication configuration :param b2sdk.v2.B2Api destination_api: B2Api instance for destination bucket; if destination bucket is on the same account as source bucket, omit this parameter and then source bucket's B2Api will be used :param b2sdk.v2.ProgressReport report: instance of ProgressReport which will report scanning progress, by default to stdout :param b2sdk.v2.ScanPoliciesManager scan_policies_manager: a strategy to scan files, so that several files that match some criteria may be omitted :rtype: b2sdk.v2.ReplicationMonitor """ bucket: Bucket rule: ReplicationRule destination_api: Optional[B2Api] = None # if None -> will use `api` of source (bucket) report: ProgressReport = field(default_factory=lambda: ProgressReport(sys.stdout, False)) scan_policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER REPORT_CLASS: ClassVar[AbstractScanReport] = ReplicationReport B2_FOLDER_CLASS: ClassVar[Type] = B2Folder QUEUE_SIZE: ClassVar[int] = 20_000 def __post_init__(self): if not self.bucket.replication: raise ValueError(f'Bucket {self.bucket} has no replication configuration') if self.rule not in self.bucket.replication.rules: raise ValueError(f'Rule {self.rule} is not a rule from {self.bucket}') @property def source_api(self) -> B2Api: return self.bucket.api @property def source_folder(self) -> B2_FOLDER_CLASS: return self.B2_FOLDER_CLASS( bucket_name=self.bucket.name, folder_name=self.rule.file_name_prefix, api=self.source_api, ) @property def destination_bucket(self) -> Bucket: destination_api = self.destination_api or self.source_api bucket_id = self.rule.destination_bucket_id return destination_api.get_bucket_by_id(bucket_id) @property def destination_folder(self) -> B2_FOLDER_CLASS: destination_bucket = self.destination_bucket return self.B2_FOLDER_CLASS( bucket_name=destination_bucket.name, folder_name=self.rule.file_name_prefix, api=destination_bucket.api, ) def iter_pairs(self) -> Iterator[Tuple[Optional[B2Path], Optional[B2Path]]]: """ Iterate over files in source and destination and yield pairs. Required for replication inspection in-depth. Return pair of (source B2Path, destination B2Path). Source or destination path may be missing if there's no corresponding destination/source file. """ yield from zip_folders( self.source_folder, self.destination_folder, reporter=self.report, policies_manager=self.scan_policies_manager, ) def scan(self, scan_destination: bool = True) -> AbstractScanReport: """ Scan source bucket (only, or with destination) and return replication report. No destination scan may give limited replication information, since it only checks files on the source bucket without checking whether they we really replicated to destination. It may be handy though if there is no access to replication destination. """ report = self.REPORT_CLASS() queue = Queue(maxsize=self.QUEUE_SIZE) if not scan_destination: def fill_queue(): for path in self.source_folder.all_files( policies_manager=self.scan_policies_manager, reporter=self.report, ): queue.put((path,), block=True) queue.put(None, block=True) else: def fill_queue(): for pair in self.iter_pairs(): queue.put(pair, block=True) queue.put(None, block=True) def consume_queue(): while True: items = queue.get(block=True) if items is None: # using None as "end of queue" marker break report.add(*items) with ThreadPoolExecutor(max_workers=2) as thread_pool: futures = [ thread_pool.submit(fill_queue), thread_pool.submit(consume_queue), ] for future in futures: future.result() return report b2-sdk-python-1.17.3/b2sdk/replication/setting.py000066400000000000000000000162431426424117700215670ustar00rootroot00000000000000###################################################################### # # File: b2sdk/replication/setting.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import re from builtins import classmethod from dataclasses import dataclass, field from typing import ClassVar, Dict, List, Optional @dataclass class ReplicationRule: """ Hold information about replication rule: destination bucket, priority, prefix and rule name. """ DEFAULT_PRIORITY: ClassVar[int] = 128 destination_bucket_id: str name: str file_name_prefix: str = '' is_enabled: bool = True priority: int = DEFAULT_PRIORITY include_existing_files: bool = False REPLICATION_RULE_REGEX: ClassVar = re.compile(r'^[a-zA-Z0-9_\-]{1,64}$') MIN_PRIORITY: ClassVar[int] = 1 MAX_PRIORITY: ClassVar[int] = 2**31 - 1 def __post_init__(self): if not self.destination_bucket_id: raise ValueError('destination_bucket_id is required') if not self.REPLICATION_RULE_REGEX.match(self.name): raise ValueError('replication rule name is invalid') if not (self.MIN_PRIORITY <= self.priority <= self.MAX_PRIORITY): raise ValueError( 'priority should be within [%d, %d] interval' % ( self.MIN_PRIORITY, self.MAX_PRIORITY, ) ) def as_dict(self) -> dict: return { 'destinationBucketId': self.destination_bucket_id, 'fileNamePrefix': self.file_name_prefix, 'includeExistingFiles': self.include_existing_files, 'isEnabled': self.is_enabled, 'priority': self.priority, 'replicationRuleName': self.name, } @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationRule': kwargs = {} for field_, protocolField in ( ('destination_bucket_id', 'destinationBucketId'), ('name', 'replicationRuleName'), ('file_name_prefix', 'fileNamePrefix'), ('include_existing_files', 'includeExistingFiles'), ('is_enabled', 'isEnabled'), ('priority', 'priority'), ): value = value_dict.get( protocolField ) # refactor to := when dropping Python 3.7, maybe even dict expression if value is not None: kwargs[field_] = value return cls(**kwargs) @dataclass class ReplicationConfiguration: """ Hold information about bucket replication configuration """ # configuration as source: rules: List[ReplicationRule] = field(default_factory=list) source_key_id: Optional[str] = None # configuration as destination: source_to_destination_key_mapping: Dict[str, str] = field(default_factory=dict) def __post_init__(self): if self.rules and not self.source_key_id: raise ValueError("source_key_id must not be empty") for source, destination in self.source_to_destination_key_mapping.items(): if not source or not destination: raise ValueError( "source_to_destination_key_mapping must not contain \ empty keys or values: ({}, {})".format(source, destination) ) @property def is_source(self) -> bool: return bool(self.source_key_id) def get_source_configuration_as_dict(self) -> dict: return { 'rules': self.rules, 'source_key_id': self.source_key_id, } @property def is_destination(self) -> bool: return bool(self.source_to_destination_key_mapping) def get_destination_configuration_as_dict(self) -> dict: return { 'source_to_destination_key_mapping': self.source_to_destination_key_mapping, } def as_dict(self) -> dict: """ Represent the setting as a dict, for example: .. code-block:: python { "asReplicationSource": { "replicationRules": [ { "destinationBucketId": "c5f35d53a90a7ea284fb0719", "fileNamePrefix": "", "includeExistingFiles": True, "isEnabled": true, "priority": 1, "replicationRuleName": "replication-us-west" }, { "destinationBucketId": "55f34d53a96a7ea284fb0719", "fileNamePrefix": "", "includeExistingFiles": True, "isEnabled": true, "priority": 2, "replicationRuleName": "replication-us-west-2" } ], "sourceApplicationKeyId": "10053d55ae26b790000000006" }, "asReplicationDestination": { "sourceToDestinationKeyMapping": { "10053d55ae26b790000000045": "10053d55ae26b790000000004", "10053d55ae26b790000000046": "10053d55ae26b790030000004" } } } """ result = { 'asReplicationSource': { "replicationRules": [rule.as_dict() for rule in self.rules], "sourceApplicationKeyId": self.source_key_id, } if self.is_source else None, 'asReplicationDestination': { 'sourceToDestinationKeyMapping': self.source_to_destination_key_mapping, } if self.is_destination else None, } return result serialize_to_json_for_request = as_dict @classmethod def from_dict(cls, value_dict: dict) -> 'ReplicationConfiguration': source_dict = value_dict.get('asReplicationSource') or {} destination_dict = value_dict.get('asReplicationDestination') or {} return cls( rules=[ ReplicationRule.from_dict(rule_dict) for rule_dict in source_dict.get('replicationRules', []) ], source_key_id=source_dict.get('sourceApplicationKeyId'), source_to_destination_key_mapping=destination_dict.get('sourceToDestinationKeyMapping') or {}, ) @dataclass class ReplicationConfigurationFactory: is_client_authorized_to_read: bool value: Optional[ReplicationConfiguration] @classmethod def from_bucket_dict(cls, bucket_dict: dict) -> 'ReplicationConfigurationFactory': """ Returns ReplicationConfigurationFactory for the given bucket dict retrieved from the api. """ replication_dict = bucket_dict.get('replicationConfiguration') or {} value_dict = replication_dict.get('value') or {} return cls( is_client_authorized_to_read=replication_dict.get('isClientAuthorizedToRead', True), value=ReplicationConfiguration.from_dict(value_dict), ) b2-sdk-python-1.17.3/b2sdk/replication/setup.py000066400000000000000000000317741426424117700212600ustar00rootroot00000000000000###################################################################### # # File: b2sdk/replication/setup.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # b2 replication-setup [--profile profileName] --destination-profile destinationProfileName sourceBucketPath destinationBucketName [ruleName] # b2 replication-debug [--profile profileName] [--destination-profile destinationProfileName] bucketPath # b2 replication-status [--profile profileName] [--destination-profile destinationProfileName] [sourceBucketPath] [destinationBucketPath] # b2 replication-pause [--profile profileName] (sourceBucketName|sourceBucketPath) [replicationRuleName] # b2 replication-unpause [--profile profileName] (sourceBucketName|sourceBucketPath) [replicationRuleName] # b2 replication-accept destinationBucketName sourceKeyId [destinationKeyId] # b2 replication-deny destinationBucketName sourceKeyId from collections.abc import Iterable from typing import ClassVar, List, Optional, Tuple import itertools import logging from b2sdk.api import B2Api from b2sdk.application_key import ApplicationKey from b2sdk.bucket import Bucket from b2sdk.utils import B2TraceMeta from b2sdk.replication.setting import ReplicationConfiguration, ReplicationRule logger = logging.getLogger(__name__) try: Iterable[str] except TypeError: Iterable = List # Remove after dropping Python 3.8 class ReplicationSetupHelper(metaclass=B2TraceMeta): """ class with various methods that help with setting up repliction """ PRIORITY_OFFSET: ClassVar[int] = 5 #: how far to to put the new rule from the existing rules DEFAULT_PRIORITY: ClassVar[ int ] = ReplicationRule.DEFAULT_PRIORITY #: what priority to set if there are no preexisting rules MAX_PRIORITY: ClassVar[ int] = ReplicationRule.MAX_PRIORITY #: maximum allowed priority of a replication rule DEFAULT_SOURCE_CAPABILITIES: ClassVar[Tuple[str, ...]] = ( 'readFiles', 'readFileLegalHolds', 'readFileRetentions', ) DEFAULT_DESTINATION_CAPABILITIES: ClassVar[Tuple[str, ...]] = ( 'writeFiles', 'writeFileLegalHolds', 'writeFileRetentions', 'deleteFiles', ) def setup_both( self, source_bucket: Bucket, destination_bucket: Bucket, name: Optional[str] = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule prefix: Optional[str] = None, include_existing_files: bool = False, ) -> Tuple[Bucket, Bucket]: # setup source key source_key = self._get_source_key( source_bucket, prefix, source_bucket.replication, ) # setup destination new_destination_bucket = self.setup_destination( source_key.id_, destination_bucket, ) # setup source new_source_bucket = self.setup_source( source_bucket, source_key, destination_bucket, prefix, name, priority, include_existing_files, ) return new_source_bucket, new_destination_bucket def setup_destination( self, source_key_id: str, destination_bucket: Bucket, ) -> Bucket: api: B2Api = destination_bucket.api # yapf: disable source_configuration = destination_bucket.replication.get_source_configuration_as_dict( ) if destination_bucket.replication else {} destination_configuration = destination_bucket.replication.get_destination_configuration_as_dict( ) if destination_bucket.replication else {'source_to_destination_key_mapping': {}} keys_to_purge, destination_key = self._get_destination_key( api, destination_bucket, ) # note: no clean up of keys_to_purge is actually done destination_configuration['source_to_destination_key_mapping'][source_key_id] = destination_key.id_ new_replication_configuration = ReplicationConfiguration( **source_configuration, **destination_configuration, ) # yapf: enable return destination_bucket.update( if_revision_is=destination_bucket.revision, replication=new_replication_configuration, ) @classmethod def _get_destination_key( cls, api: B2Api, destination_bucket: Bucket, ): keys_to_purge = [] if destination_bucket.replication is not None: current_destination_key_ids = destination_bucket.replication.source_to_destination_key_mapping.values() # yapf: disable else: current_destination_key_ids = [] key = None for current_destination_key_id in current_destination_key_ids: # potential inefficiency here as we are fetching keys one by one, however # the number of keys on an account is limited to a 100 000 000 per account lifecycle # while the number of keys in the map can be expected to be very low current_destination_key = api.get_key(current_destination_key_id) if current_destination_key is None: logger.debug( 'zombie key found in replication destination_configuration.source_to_destination_key_mapping: %s', current_destination_key_id ) keys_to_purge.append(current_destination_key_id) continue if current_destination_key.has_capabilities( cls.DEFAULT_DESTINATION_CAPABILITIES ) and not current_destination_key.name_prefix: logger.debug('matching destination key found: %s', current_destination_key_id) key = current_destination_key # not breaking here since we want to fill the purge list else: logger.info('non-matching destination key found: %s', current_destination_key) if not key: logger.debug("no matching key found, making a new one") key = cls._create_destination_key( name=destination_bucket.name[:91] + '-replidst', bucket=destination_bucket, prefix=None, ) return keys_to_purge, key def setup_source( self, source_bucket: Bucket, source_key: ApplicationKey, destination_bucket: Bucket, prefix: Optional[str] = None, name: Optional[str] = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule include_existing_files: bool = False, ) -> Bucket: if prefix is None: prefix = "" if source_bucket.replication: current_source_rules = source_bucket.replication.rules destination_configuration = source_bucket.replication.get_destination_configuration_as_dict( ) else: current_source_rules = [] destination_configuration = {} priority = self._get_priority_for_new_rule( current_source_rules, priority, ) name = self._get_new_rule_name( current_source_rules, destination_bucket, name, ) new_rr = ReplicationRule( name=name, priority=priority, destination_bucket_id=destination_bucket.id_, file_name_prefix=prefix, include_existing_files=include_existing_files, ) new_replication_configuration = ReplicationConfiguration( source_key_id=source_key.id_, rules=current_source_rules + [new_rr], **destination_configuration, ) return source_bucket.update( if_revision_is=source_bucket.revision, replication=new_replication_configuration, ) @classmethod def _get_source_key( cls, source_bucket: Bucket, prefix: str, current_replication_configuration: ReplicationConfiguration, ) -> ApplicationKey: api = source_bucket.api if current_replication_configuration is not None: current_source_key = api.get_key(current_replication_configuration.source_key_id) do_create_key = cls._should_make_new_source_key( current_replication_configuration, current_source_key, ) if not do_create_key: return current_source_key new_key = cls._create_source_key( name=source_bucket.name[:91] + '-replisrc', bucket=source_bucket, prefix=prefix, ) return new_key @classmethod def _should_make_new_source_key( cls, current_replication_configuration: ReplicationConfiguration, current_source_key: Optional[ApplicationKey], ) -> bool: if current_replication_configuration.source_key_id is None: logger.debug('will create a new source key because no key is set') return True if current_source_key is None: logger.debug( 'will create a new source key because current key "%s" was deleted', current_replication_configuration.source_key_id, ) return True if current_source_key.name_prefix: logger.debug( 'will create a new source key because current key %s had a prefix: "%s"', current_source_key.name_prefix, ) return True if not current_source_key.has_capabilities(cls.DEFAULT_SOURCE_CAPABILITIES): logger.debug( 'will create a new source key because %s installed so far does not have enough permissions for replication source: ', current_source_key.id_, current_source_key.capabilities, ) return True return False # current key is ok @classmethod def _create_source_key( cls, name: str, bucket: Bucket, prefix: Optional[str] = None, ) -> ApplicationKey: # in this implementation we ignore the prefix and create a full key, because # if someone would need a different (wider) key later, all replication # destinations would have to start using new keys and it's not feasible # from organizational perspective, while the prefix of uploaded files can be # restricted on the rule level prefix = None capabilities = cls.DEFAULT_SOURCE_CAPABILITIES return cls._create_key(name, bucket, prefix, capabilities) @classmethod def _create_destination_key( cls, name: str, bucket: Bucket, prefix: Optional[str] = None, ) -> ApplicationKey: capabilities = cls.DEFAULT_DESTINATION_CAPABILITIES return cls._create_key(name, bucket, prefix, capabilities) @classmethod def _create_key( cls, name: str, bucket: Bucket, prefix: Optional[str] = None, capabilities=tuple(), ) -> ApplicationKey: api: B2Api = bucket.api return api.create_key( capabilities=capabilities, key_name=name, bucket_id=bucket.id_, name_prefix=prefix, ) @classmethod def _get_priority_for_new_rule( cls, current_rules: Iterable[ReplicationRule], priority: Optional[int] = None, ): if priority is not None: return priority if current_rules: # ignore a case where the existing rrs need to have their priorities decreased to make space (max is 2**31-1) existing_priority = max(rr.priority for rr in current_rules) return min(existing_priority + cls.PRIORITY_OFFSET, cls.MAX_PRIORITY) return cls.DEFAULT_PRIORITY @classmethod def _get_new_rule_name( cls, current_rules: Iterable[ReplicationRule], destination_bucket: Bucket, name: Optional[str] = None, ): if name is not None: return name existing_names = set(rr.name for rr in current_rules) suffixes = cls._get_rule_name_candidate_suffixes() while True: candidate = '%s%s' % ( destination_bucket.name, next(suffixes), ) # use := after dropping 3.7 if candidate not in existing_names: return candidate @classmethod def _get_rule_name_candidate_suffixes(cls): """ >>> a = ReplicationSetupHelper._get_rule_name_candidate_suffixes() >>> [next(a) for i in range(10)] ['', '2', '3', '4', '5', '6', '7', '8', '9', '10'] """ return map(str, itertools.chain([''], itertools.count(2))) @classmethod def _partion_bucket_path(cls, bucket_path: str) -> Tuple[str, str]: bucket_name, _, path = bucket_path.partition('/') return bucket_name, path b2-sdk-python-1.17.3/b2sdk/replication/types.py000066400000000000000000000012721426424117700212520ustar00rootroot00000000000000###################################################################### # # File: b2sdk/replication/types.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from enum import Enum, unique from typing import Optional @unique class ReplicationStatus(Enum): PENDING = 'PENDING' COMPLETED = 'COMPLETED' FAILED = 'FAILED' REPLICA = 'REPLICA' @classmethod def from_response_headers(cls, headers: dict) -> Optional['ReplicationStatus']: value = headers.get('X-Bz-Replication-Status', None) return value and cls[value.upper()] b2-sdk-python-1.17.3/b2sdk/requests/000077500000000000000000000000001426424117700170745ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/requests/LICENSE000066400000000000000000000236351426424117700201120ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.b2-sdk-python-1.17.3/b2sdk/requests/NOTICE000066400000000000000000000004411426424117700177770ustar00rootroot00000000000000Requests Copyright 2019 Kenneth Reitz Copyright 2021 Backblaze Inc. Changes made to the original source: requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` in order to NOT decompress data based on Content-Encoding headerb2-sdk-python-1.17.3/b2sdk/requests/README.md000066400000000000000000000003501426424117700203510ustar00rootroot00000000000000This module contains modified parts of the requests module (https://github.com/psf/requests). The modules original license is included in LICENSE. Changes made to the original source are listed in NOTICE, along with original NOTICE.b2-sdk-python-1.17.3/b2sdk/requests/__init__.py000066400000000000000000000062671426424117700212200ustar00rootroot00000000000000###################################################################### # # File: b2sdk/requests/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # Copyright 2019 Kenneth Reitz # # License https://www.backblaze.com/using_b2_code.html # License Apache License 2.0 (http://www.apache.org/licenses/ and LICENSE file in this directory) # ###################################################################### """This file contains modified parts of the requests module (https://github.com/psf/requests, models.py), original Copyright 2019 Kenneth Reitz Changes made to the original source: see NOTICE """ from requests import Response, ConnectionError from requests.exceptions import ChunkedEncodingError, ContentDecodingError, StreamConsumedError from requests.utils import iter_slices, stream_decode_response_unicode from urllib3.exceptions import ProtocolError, DecodeError, ReadTimeoutError from . import included_source_meta class NotDecompressingResponse(Response): def iter_content(self, chunk_size=1, decode_unicode=False): def generate(): # Special case for urllib3. if hasattr(self.raw, 'stream'): try: # set decode_content to False to prevent decompressing files that # when Content-Encoding response header is set for chunk in self.raw.stream(chunk_size, decode_content=False): yield chunk except ProtocolError as e: raise ChunkedEncodingError(e) except DecodeError as e: raise ContentDecodingError(e) except ReadTimeoutError as e: raise ConnectionError(e) else: # Standard file-like object. while True: chunk = self.raw.read(chunk_size) if not chunk: break yield chunk self._content_consumed = True if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() elif chunk_size is not None and not isinstance(chunk_size, int): raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) stream_chunks = generate() chunks = reused_chunks if self._content_consumed else stream_chunks if decode_unicode: chunks = stream_decode_response_unicode(chunks, self) return chunks @classmethod def from_builtin_response(cls, response: Response): """ Create a :class:`b2sdk.requests.NotDecompressingResponse` object from a :class:`requests.Response` object. Don't use :code:`Response.__getstate__` and :code:`Response.__setstate__` because these assume that the content has been consumed, which will never be true in our case. """ new_response = cls() for attr_name in cls.__attrs__: setattr(new_response, attr_name, getattr(response, attr_name)) new_response.raw = response.raw return new_response b2-sdk-python-1.17.3/b2sdk/requests/included_source_meta.py000066400000000000000000000015201426424117700236210ustar00rootroot00000000000000###################################################################### # # File: b2sdk/requests/included_source_meta.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.included_sources import add_included_source, IncludedSourceMeta included_source_meta = IncludedSourceMeta( 'requests', 'Included in a revised form', { 'NOTICE': """Requests Copyright 2019 Kenneth Reitz Copyright 2021 Backblaze Inc. Changes made to the original source: requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` in order to NOT decompress data based on Content-Encoding header""" } ) add_included_source(included_source_meta) b2-sdk-python-1.17.3/b2sdk/scan/000077500000000000000000000000001426424117700161455ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/scan/__init__.py000066400000000000000000000004411426424117700202550ustar00rootroot00000000000000###################################################################### # # File: b2sdk/scan/__init__.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/scan/exception.py000066400000000000000000000054411426424117700205210ustar00rootroot00000000000000###################################################################### # # File: b2sdk/scan/exception.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from contextlib import contextmanager from typing import Iterator, Type from ..exception import B2Error, B2SimpleError class EnvironmentEncodingError(B2Error): """ Raised when a file name can not be decoded with system encoding. """ def __init__(self, filename, encoding): """ :param filename: an encoded file name :type filename: str, bytes :param str encoding: file name encoding """ super().__init__() self.filename = filename self.encoding = encoding def __str__(self): return """file name %s cannot be decoded with system encoding (%s). We think this is an environment error which you should workaround by setting your system encoding properly, for example like this: export LANG=en_US.UTF-8""" % ( self.filename, self.encoding, ) class InvalidArgument(B2Error): """ Raised when one or more arguments are invalid """ def __init__(self, parameter_name, message): """ :param parameter_name: name of the function argument :param message: brief explanation of misconfiguration """ super().__init__() self.parameter_name = parameter_name self.message = message def __str__(self): return "%s %s" % (self.parameter_name, self.message) class UnsupportedFilename(B2Error): """ Raised when a filename is not supported by the scan operation """ def __init__(self, message, filename): """ :param message: brief explanation of why the filename was not supported :param filename: name of the file which is not supported """ super().__init__() self.filename = filename self.message = message def __str__(self): return "%s: %s" % (self.message, self.filename) @contextmanager def check_invalid_argument(parameter_name: str, message: str, *exceptions: Type[Exception]) -> Iterator[None]: """Raise `InvalidArgument` in case of one of given exception was thrown.""" try: yield except exceptions as exc: if not message: message = str(exc) raise InvalidArgument(parameter_name, message) from exc class BaseDirectoryError(B2SimpleError): def __init__(self, path): self.path = path super().__init__(path) class EmptyDirectory(BaseDirectoryError): pass class UnableToCreateDirectory(BaseDirectoryError): pass class NotADirectory(BaseDirectoryError): pass b2-sdk-python-1.17.3/b2sdk/scan/folder.py000066400000000000000000000342461426424117700200030ustar00rootroot00000000000000###################################################################### # # File: b2sdk/scan/folder.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging import os import platform import re import sys from abc import ABCMeta, abstractmethod from typing import Iterator from ..utils import fix_windows_path_limit, get_file_mtime, is_file_readable from .exception import EmptyDirectory, EnvironmentEncodingError, NotADirectory, UnableToCreateDirectory, UnsupportedFilename from .path import AbstractPath, B2Path, LocalPath from .policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from .report import ProgressReport DRIVE_MATCHER = re.compile(r"^([A-Za-z]):([/\\])") ABSOLUTE_PATH_MATCHER = re.compile(r"^(/)|^(\\)") RELATIVE_PATH_MATCHER = re.compile( # "abc" and "xyz" represent anything, including "nothing" r"^(\.\.[/\\])|" + # ../abc or ..\abc r"^(\.[/\\])|" + # ./abc or .\abc r"([/\\]\.\.[/\\])|" + # abc/../xyz or abc\..\xyz or abc\../xyz or abc/..\xyz r"([/\\]\.[/\\])|" + # abc/./xyz or abc\.\xyz or abc\./xyz or abc/.\xyz r"([/\\]\.\.)$|" + # abc/.. or abc\.. r"([/\\]\.)$|" + # abc/. or abc\. r"^(\.\.)$|" + # just ".." r"([/\\][/\\])|" + # abc\/xyz or abc/\xyz or abc//xyz or abc\\xyz r"^(\.)$" # just "." ) # yapf: disable logger = logging.getLogger(__name__) class AbstractFolder(metaclass=ABCMeta): """ Interface to a folder full of files, which might be a B2 bucket, a virtual folder in a B2 bucket, or a directory on a local file system. Files in B2 may have multiple versions, while files in local folders have just one. """ @abstractmethod def all_files(self, reporter: ProgressReport, policies_manager=DEFAULT_SCAN_MANAGER) -> Iterator[AbstractPath]: """ Return an iterator over all of the files in the folder, in the order that B2 uses. It also performs filtering using policies manager. No matter what the folder separator on the local file system is, "/" is used in the returned file names. If a file is found, but does not exist (for example due to a broken symlink or a race), reporter will be informed about each such problem. :param reporter: a place to report errors :param policies_manager: a policies manager object """ @abstractmethod def folder_type(self): """ Return one of: 'b2', 'local'. :rtype: str """ @abstractmethod def make_full_path(self, file_name): """ Return the full path to the file. :param file_name: a file name :type file_name: str :rtype: str """ def join_b2_path(relative_dir_path: str, file_name: str): """ Like os.path.join, but for B2 file names where the root directory is called ''. """ if relative_dir_path == '': return file_name else: return relative_dir_path + '/' + file_name class LocalFolder(AbstractFolder): """ Folder interface to a directory on the local machine. """ def __init__(self, root): """ Initialize a new folder. :param root: path to the root of the local folder. Must be unicode. :type root: str """ if not isinstance(root, str): raise ValueError('folder path should be unicode: %s' % repr(root)) self.root = fix_windows_path_limit(os.path.abspath(root)) def folder_type(self): """ Return folder type. :rtype: str """ return 'local' def all_files(self, reporter: ProgressReport, policies_manager=DEFAULT_SCAN_MANAGER) -> Iterator[LocalPath]: """ Yield all files. :param reporter: a place to report errors :param policies_manager: a policy manager object, default is DEFAULT_SCAN_MANAGER """ yield from self._walk_relative_paths(self.root, '', reporter, policies_manager) def make_full_path(self, file_name): """ Convert a file name into an absolute path, ensure it is not outside self.root :param file_name: a file name :type file_name: str """ # Fix OS path separators file_name = file_name.replace('/', os.path.sep) # Generate the full path to the file full_path = os.path.normpath(os.path.join(self.root, file_name)) # Get the common prefix between the new full_path and self.root common_prefix = os.path.commonprefix([full_path, self.root]) # Ensure the new full_path is inside the self.root directory if common_prefix != self.root: raise UnsupportedFilename("illegal file name", full_path) return full_path def ensure_present(self): """ Make sure that the directory exists. """ if not os.path.exists(self.root): try: os.mkdir(self.root) except OSError: raise UnableToCreateDirectory(self.root) elif not os.path.isdir(self.root): raise NotADirectory(self.root) def ensure_non_empty(self): """ Make sure that the directory exists and is non-empty. """ self.ensure_present() if not os.listdir(self.root): raise EmptyDirectory(self.root) def _walk_relative_paths( self, local_dir: str, relative_dir_path: str, reporter, policies_manager: ScanPoliciesManager ): """ Yield a File object for each of the files anywhere under this folder, in the order they would appear in B2, unless the path is excluded by policies manager. :param relative_dir_path: the path of this dir relative to the scan point, or '' if at scan point """ if not isinstance(local_dir, str): raise ValueError('folder path should be unicode: %s' % repr(local_dir)) # Collect the names. We do this before returning any results, because # directories need to sort as if their names end in '/'. # # With a directory containing 'a', 'a.txt', and 'a0.txt', with 'a' being # a directory containing 'b.txt', and 'c.txt', the results returned # should be: # # a.txt # a/b.txt # a/c.txt # a0.txt # # This is because in Unicode '.' comes before '/', which comes before '0'. names = [] # list of (name, local_path, relative_file_path) for name in os.listdir(local_dir): # We expect listdir() to return unicode if dir_path is unicode. # If the file name is not valid, based on the file system # encoding, then listdir() will return un-decoded str/bytes. if not isinstance(name, str): name = self._handle_non_unicode_file_name(name) if '/' in name: raise UnsupportedFilename( "scan does not support file names that include '/'", "%s in dir %s" % (name, local_dir) ) local_path = os.path.join(local_dir, name) relative_file_path = join_b2_path( relative_dir_path, name ) # file path relative to the scan point # Skip broken symlinks or other inaccessible files if not is_file_readable(local_path, reporter): continue if policies_manager.exclude_all_symlinks and os.path.islink(local_path): if reporter is not None: reporter.symlink_skipped(local_path) continue if os.path.isdir(local_path): name += '/' if policies_manager.should_exclude_local_directory(relative_file_path): continue names.append((name, local_path, relative_file_path)) # Yield all of the answers. # # Sorting the list of triples puts them in the right order because 'name', # the sort key, is the first thing in the triple. for (name, local_path, relative_file_path) in sorted(names): if name.endswith('/'): for subdir_file in self._walk_relative_paths( local_path, relative_file_path, reporter, policies_manager ): yield subdir_file else: # Check that the file still exists and is accessible, since it can take a long time # to iterate through large folders if is_file_readable(local_path, reporter): file_mod_time = get_file_mtime(local_path) file_size = os.path.getsize(local_path) local_scan_path = LocalPath( absolute_path=self.make_full_path(relative_file_path), relative_path=relative_file_path, mod_time=file_mod_time, size=file_size, ) if policies_manager.should_exclude_local_path(local_scan_path): continue yield local_scan_path @classmethod def _handle_non_unicode_file_name(cls, name): """ Decide what to do with a name returned from os.listdir() that isn't unicode. We think that this only happens when the file name can't be decoded using the file system encoding. Just in case that's not true, we'll allow all-ascii names. """ # if it's all ascii, allow it if all(b <= 127 for b in name): return name raise EnvironmentEncodingError(repr(name), sys.getfilesystemencoding()) def __repr__(self): return 'LocalFolder(%s)' % (self.root,) def b2_parent_dir(file_name): # Various Parent dir getting method have been tested, and this one seems to be the faste # After dropping python 3.9 support: refactor this use the "match" syntax try: dir_name, _ = file_name.rsplit('/', 1) except ValueError: return '' return dir_name class B2Folder(AbstractFolder): """ Folder interface to b2. """ def __init__(self, bucket_name, folder_name, api): """ :param bucket_name: a name of the bucket :type bucket_name: str :param folder_name: a folder name :type folder_name: str :param api: an API object :type api: b2sdk.api.B2Api """ self.bucket_name = bucket_name self.folder_name = folder_name self.bucket = api.get_bucket_by_name(bucket_name) self.api = api self.prefix = self.folder_name if self.prefix and self.prefix[-1] != '/': self.prefix += '/' def all_files( self, reporter: ProgressReport, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER ) -> Iterator[B2Path]: """ Yield all files. """ current_name = None last_ignored_dir = None current_versions = [] current_file_version = None for file_version in self.get_file_versions(): if current_file_version is None: current_file_version = file_version assert file_version.file_name.startswith(self.prefix) if file_version.action == 'start': continue file_name = file_version.file_name[len(self.prefix):] if last_ignored_dir is not None and file_name.startswith(last_ignored_dir): continue dir_name = b2_parent_dir(file_name) if policies_manager.should_exclude_b2_directory(dir_name): last_ignored_dir = dir_name + '/' continue else: last_ignored_dir = None if policies_manager.should_exclude_b2_file_version(file_version, file_name): continue self._validate_file_name(file_name) if current_name != file_name and current_name is not None and current_versions: yield B2Path( relative_path=current_name, selected_version=current_versions[0], all_versions=current_versions ) current_versions = [] current_name = file_name current_versions.append(file_version) if current_name is not None and current_versions: yield B2Path( relative_path=current_name, selected_version=current_versions[0], all_versions=current_versions ) def get_file_versions(self): for file_version, _ in self.bucket.ls( self.folder_name, latest_only=False, recursive=True, ): yield file_version def _validate_file_name(self, file_name): # Do not allow relative paths in file names if RELATIVE_PATH_MATCHER.search(file_name): raise UnsupportedFilename( "scan does not support file names that include relative paths", file_name ) # Do not allow absolute paths in file names if ABSOLUTE_PATH_MATCHER.search(file_name): raise UnsupportedFilename( "scan does not support file names with absolute paths", file_name ) # On Windows, do not allow drive letters in file names if platform.system() == "Windows" and DRIVE_MATCHER.search(file_name): raise UnsupportedFilename( "scan does not support file names with drive letters", file_name ) def folder_type(self): """ Return folder type. :rtype: str """ return 'b2' def make_full_path(self, file_name): """ Make an absolute path from a file name. :param file_name: a file name :type file_name: str """ if self.folder_name == '': return file_name else: return self.folder_name + '/' + file_name def __str__(self): return 'B2Folder(%s, %s)' % (self.bucket_name, self.folder_name) b2-sdk-python-1.17.3/b2sdk/scan/folder_parser.py000066400000000000000000000036671426424117700213620ustar00rootroot00000000000000###################################################################### # # File: b2sdk/scan/folder_parser.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from .exception import InvalidArgument from .folder import B2Folder, LocalFolder def parse_folder(folder_name, api, local_folder_class=LocalFolder, b2_folder_class=B2Folder): """ Take either a local path, or a B2 path, and returns a Folder object for it. B2 paths look like: b2://bucketName/path/name. The '//' is optional. Anything else is treated like a local folder. :param folder_name: a name of the folder, either local or remote :type folder_name: str :param api: an API object :type api: :class:`~b2sdk.v2.B2Api` :param local_folder_class: class to handle local folders :type local_folder_class: `~b2sdk.v2.AbstractFolder` :param b2_folder_class: class to handle B2 folders :type b2_folder_class: `~b2sdk.v2.AbstractFolder` """ if folder_name.startswith('b2://'): return _parse_bucket_and_folder(folder_name[5:], api, b2_folder_class) elif folder_name.startswith('b2:') and folder_name[3].isalnum(): return _parse_bucket_and_folder(folder_name[3:], api, b2_folder_class) else: return local_folder_class(folder_name) def _parse_bucket_and_folder(bucket_and_path, api, b2_folder_class): """ Turn 'my-bucket/foo' into B2Folder(my-bucket, foo). """ if '//' in bucket_and_path: raise InvalidArgument('folder_name', "'//' not allowed in path names") if '/' not in bucket_and_path: bucket_name = bucket_and_path folder_name = '' else: (bucket_name, folder_name) = bucket_and_path.split('/', 1) if folder_name.endswith('/'): folder_name = folder_name[:-1] return b2_folder_class(bucket_name, folder_name, api)b2-sdk-python-1.17.3/b2sdk/scan/path.py000066400000000000000000000053151426424117700174570ustar00rootroot00000000000000###################################################################### # # File: b2sdk/scan/path.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import ABC, abstractmethod from typing import List from ..file_version import FileVersion class AbstractPath(ABC): """ Represent a path in a source or destination folder - be it B2 or local """ def __init__(self, relative_path: str, mod_time: int, size: int): self.relative_path = relative_path self.mod_time = mod_time self.size = size @abstractmethod def is_visible(self) -> bool: """Is the path visible/not deleted on it's storage""" def __repr__(self): return '%s(%s, %s, %s)' % ( self.__class__.__name__, repr(self.relative_path), repr(self.mod_time), repr(self.size) ) class LocalPath(AbstractPath): __slots__ = ['absolute_path', 'relative_path', 'mod_time', 'size'] def __init__(self, absolute_path: str, relative_path: str, mod_time: int, size: int): self.absolute_path = absolute_path super().__init__(relative_path, mod_time, size) def is_visible(self) -> bool: return True def __eq__(self, other): return ( self.absolute_path == other.absolute_path and self.relative_path == other.relative_path and self.mod_time == other.mod_time and self.size == other.size ) class B2Path(AbstractPath): __slots__ = ['relative_path', 'selected_version', 'all_versions'] def __init__( self, relative_path: str, selected_version: FileVersion, all_versions: List[FileVersion] ): self.selected_version = selected_version self.all_versions = all_versions self.relative_path = relative_path def is_visible(self) -> bool: return self.selected_version.action != 'hide' @property def mod_time(self) -> int: return self.selected_version.mod_time_millis @property def size(self) -> int: return self.selected_version.size def __repr__(self): return '%s(%s, [%s])' % ( self.__class__.__name__, self.relative_path, ', '.join( '(%s, %s, %s)' % ( repr(fv.id_), repr(fv.mod_time_millis), repr(fv.action), ) for fv in self.all_versions ) ) def __eq__(self, other): return ( self.relative_path == other.relative_path and self.selected_version == other.selected_version and self.all_versions == other.all_versions )b2-sdk-python-1.17.3/b2sdk/scan/policies.py000066400000000000000000000213131426424117700203260ustar00rootroot00000000000000###################################################################### # # File: b2sdk/scan/policies.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging import re from typing import Iterable, Optional, Union from ..file_version import FileVersion from .exception import InvalidArgument, check_invalid_argument from .path import LocalPath logger = logging.getLogger(__name__) class RegexSet: """ Hold a (possibly empty) set of regular expressions and know how to check whether a string matches any of them. """ def __init__(self, regex_iterable): """ :param regex_iterable: an interable which yields regexes """ self._compiled_list = [re.compile(r) for r in regex_iterable] def matches(self, s): """ Check whether a string matches any of regular expressions. :param s: a string to check :type s: str :rtype: bool """ return any(c.match(s) is not None for c in self._compiled_list) def convert_dir_regex_to_dir_prefix_regex(dir_regex): """ The patterns used to match directory names (and file names) are allowed to match a prefix of the name. This 'feature' was unintentional, but is being retained for compatibility. This means that a regex that matches a directory name can't be used directly to match against a file name and test whether the file should be excluded because it matches the directory. The pattern 'photos' will match directory names 'photos' and 'photos2', and should exclude files 'photos/kitten.jpg', and 'photos2/puppy.jpg'. It should not exclude 'photos.txt', because there is no directory name that matches. On the other hand, the pattern 'photos$' should match 'photos/kitten.jpg', but not 'photos2/puppy.jpg', nor 'photos.txt' If the original regex is valid, there are only two cases to consider: either the regex ends in '$' or does not. :param dir_regex: a regular expression string or literal :type dir_regex: str """ if dir_regex.endswith('$'): return dir_regex[:-1] + r'/' else: return dir_regex + r'.*?/' class IntegerRange: """ Hold a range of two integers. If the range value is None, it indicates that the value should be treated as -Inf (for begin) or +Inf (for end). """ def __init__(self, begin, end): """ :param begin: begin position of the range (included) :type begin: int :param end: end position of the range (included) :type end: int """ self._begin = begin self._end = end if self._begin and self._begin < 0: raise ValueError('begin time can not be less than 0, use None for the infinity') if self._end and self._end < 0: raise ValueError('end time can not be less than 0, use None for the infinity') def __contains__(self, item): ge_begin, le_end = True, True if self._begin is not None: ge_begin = item >= self._begin if self._end is not None: le_end = item <= self._end return ge_begin and le_end class ScanPoliciesManager: """ Policy object used when scanning folders, used to decide which files to include in the list of files. Code that scans through files should at least use should_exclude_file() to decide whether each file should be included; it will check include/exclude patterns for file names, as well as patterns for excluding directories. Code that scans may optionally use should_exclude_directory() to test whether it can skip a directory completely and not bother listing the files and sub-directories in it. """ def __init__( self, exclude_dir_regexes: Iterable[Union[str, re.Pattern]] = tuple(), exclude_file_regexes: Iterable[Union[str, re.Pattern]] = tuple(), include_file_regexes: Iterable[Union[str, re.Pattern]] = tuple(), exclude_all_symlinks: bool = False, exclude_modified_before: Optional[int] = None, exclude_modified_after: Optional[int] = None, exclude_uploaded_before: Optional[int] = None, exclude_uploaded_after: Optional[int] = None, ): """ :param exclude_dir_regexes: regexes to exclude directories :param exclude_file_regexes: regexes to exclude files :param include_file_regexes: regexes to include files :param exclude_all_symlinks: if True, exclude all symlinks :param exclude_modified_before: optionally exclude file versions (both local and b2) modified before (in millis) :param exclude_modified_after: optionally exclude file versions (both local and b2) modified after (in millis) :param exclude_uploaded_before: optionally exclude b2 file versions uploaded before (in millis) :param exclude_uploaded_after: optionally exclude b2 file versions uploaded after (in millis) The regex matching priority for a given path is: 1) the path is always excluded if it's dir matches `exclude_dir_regexes`, if not then 2) the path is always included if it matches `include_file_regexes`, if not then 3) the path is excluded if it matches `exclude_file_regexes`, if not then 4) the path is included """ if include_file_regexes and not exclude_file_regexes: raise InvalidArgument( 'include_file_regexes', 'cannot be used without exclude_file_regexes at the same time' ) with check_invalid_argument( 'exclude_dir_regexes', 'wrong regex was given for excluding directories', re.error ): self._exclude_dir_set = RegexSet(exclude_dir_regexes) self._exclude_file_because_of_dir_set = RegexSet( map(convert_dir_regex_to_dir_prefix_regex, exclude_dir_regexes) ) with check_invalid_argument( 'exclude_file_regexes', 'wrong regex was given for excluding files', re.error ): self._exclude_file_set = RegexSet(exclude_file_regexes) with check_invalid_argument( 'include_file_regexes', 'wrong regex was given for including files', re.error ): self._include_file_set = RegexSet(include_file_regexes) self.exclude_all_symlinks = exclude_all_symlinks with check_invalid_argument( 'exclude_modified_before,exclude_modified_after', '', ValueError ): self._include_mod_time_range = IntegerRange( exclude_modified_before, exclude_modified_after ) with check_invalid_argument( 'exclude_uploaded_before,exclude_uploaded_after', '', ValueError ): self._include_upload_time_range = IntegerRange( exclude_uploaded_before, exclude_uploaded_after ) def _should_exclude_relative_path(self, relative_path: str): if self._include_file_set.matches(relative_path): return False return self._exclude_file_set.matches(relative_path) def should_exclude_local_path(self, local_path: LocalPath): """ Whether a local path should be excluded from the scan or not. This method assumes that the directory holding the `path_` has already been checked for exclusion. """ if local_path.mod_time not in self._include_mod_time_range: return True return self._should_exclude_relative_path(local_path.relative_path) def should_exclude_b2_file_version(self, file_version: FileVersion, relative_path: str): """ Whether a b2 file version should be excluded from the scan or not. This method assumes that the directory holding the `path_` has already been checked for exclusion. """ if file_version.upload_timestamp not in self._include_upload_time_range: return True if file_version.mod_time_millis not in self._include_mod_time_range: return True return self._should_exclude_relative_path(relative_path) def should_exclude_b2_directory(self, dir_path: str): """ Given the path of a directory, relative to the scan point, decide if all of the files in it should be excluded from the scan. """ return self._exclude_dir_set.matches(dir_path) def should_exclude_local_directory(self, dir_path: str): """ Given the path of a directory, relative to the scan point, decide if all of the files in it should be excluded from the scan. """ return self._exclude_dir_set.matches(dir_path) DEFAULT_SCAN_MANAGER = ScanPoliciesManager() b2-sdk-python-1.17.3/b2sdk/scan/report.py000066400000000000000000000133411426424117700200340ustar00rootroot00000000000000###################################################################### # # File: b2sdk/scan/report.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging import threading import time from dataclasses import dataclass from io import TextIOWrapper from ..utils import format_and_scale_number logger = logging.getLogger(__name__) @dataclass class ProgressReport: """ Handle reporting progress. This class is THREAD SAFE, so it can be used from parallel scan threads. """ # Minimum time between displayed updates UPDATE_INTERVAL = 0.1 stdout: TextIOWrapper # standard output file object no_progress: bool # if True, do not show progress def __post_init__(self): self.start_time = time.time() self.count = 0 self.total_done = False self.total_count = 0 self.closed = False self.lock = threading.Lock() self.current_line = '' self.encoding_warning_was_already_printed = False self._last_update_time = 0 self._update_progress() self.warnings = [] def close(self): """ Perform a clean-up. """ with self.lock: if not self.no_progress: self._print_line('', False) self.closed = True for warning in self.warnings: self._print_line(warning, True) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def error(self, message): """ Print an error, gracefully interleaving it with a progress bar. :param message: an error message :type message: str """ self.print_completion(message) def print_completion(self, message): """ Remove the progress bar, prints a message, and puts the progress bar back. :param message: an error message :type message: str """ with self.lock: self._print_line(message, True) self._last_update_time = 0 self._update_progress() def update_count(self, delta: int): """ Report that items have been processed. """ with self.lock: self.count += delta self._update_progress() def _update_progress(self): if self.closed or self.no_progress: return now = time.time() interval = now - self._last_update_time if interval < self.UPDATE_INTERVAL: return self._last_update_time = now time_delta = time.time() - self.start_time rate = 0 if time_delta == 0 else int(self.count / time_delta) message = ' count: %d/%d %s' % ( self.count, self.total_count, format_and_scale_number(rate, '/s') ) # yapf: disable self._print_line(message, False) def _print_line(self, line, newline): """ Print a line to stdout. :param line: a string without a \r or \n in it. :type line: str :param newline: True if the output should move to a new line after this one. :type newline: bool """ if len(line) < len(self.current_line): line += ' ' * (len(self.current_line) - len(line)) try: self.stdout.write(line) except UnicodeEncodeError as encode_error: if not self.encoding_warning_was_already_printed: self.encoding_warning_was_already_printed = True self.stdout.write( '!WARNING! this terminal cannot properly handle progress reporting. encoding is %s.\n' % (self.stdout.encoding,) ) self.stdout.write(line.encode('ascii', 'backslashreplace').decode()) logger.warning( 'could not output the following line with encoding %s on stdout due to %s: %s' % (self.stdout.encoding, encode_error, line) ) if newline: self.stdout.write('\n') self.current_line = '' else: self.stdout.write('\r') self.current_line = line self.stdout.flush() def update_total(self, delta): """ Report that more files have been found for comparison. :param delta: number of files found since the last check :type delta: int """ with self.lock: self.total_count += delta self._update_progress() def end_total(self): """ Total files count is done. Can proceed to step 2. """ with self.lock: self.total_done = True self._update_progress() def local_access_error(self, path): """ Add a file access error message to the list of warnings. :param path: file path :type path: str """ self.warnings.append('WARNING: %s could not be accessed (broken symlink?)' % (path,)) def local_permission_error(self, path): """ Add a permission error message to the list of warnings. :param path: file path :type path: str """ self.warnings.append( 'WARNING: %s could not be accessed (no permissions to read?)' % (path,) ) def symlink_skipped(self, path): pass def sample_report_run(): """ Generate a sample report. """ import sys report = ProgressReport(sys.stdout, False) for i in range(20): report.update_total(1) time.sleep(0.2) if i % 2 == 0: report.update_count(1) report.end_total() report.close() b2-sdk-python-1.17.3/b2sdk/scan/scan.py000066400000000000000000000075141426424117700174520ustar00rootroot00000000000000###################################################################### # # File: b2sdk/scan/scan.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import ABCMeta, abstractclassmethod, abstractmethod from collections import Counter from dataclasses import dataclass, field from typing import ClassVar, Dict, Optional, Tuple, Type from ..file_version import FileVersion from .folder import AbstractFolder from .path import AbstractPath from .policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from .report import ProgressReport def zip_folders( folder_a: AbstractFolder, folder_b: AbstractFolder, reporter: ProgressReport, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER, ) -> Tuple[Optional[AbstractPath], Optional[AbstractPath]]: """ Iterate over all of the files in the union of two folders, matching file names. Each item is a pair (file_a, file_b) with the corresponding file in both folders. Either file (but not both) will be None if the file is in only one folder. :param b2sdk.scan.folder.AbstractFolder folder_a: first folder object. :param b2sdk.scan.folder.AbstractFolder folder_b: second folder object. :param reporter: reporter object :param policies_manager: policies manager object :return: yields two element tuples """ iter_a = folder_a.all_files(reporter, policies_manager) iter_b = folder_b.all_files(reporter) current_a = next(iter_a, None) current_b = next(iter_b, None) while current_a is not None or current_b is not None: if current_a is None: yield (None, current_b) current_b = next(iter_b, None) elif current_b is None: yield (current_a, None) current_a = next(iter_a, None) elif current_a.relative_path < current_b.relative_path: yield (current_a, None) current_a = next(iter_a, None) elif current_b.relative_path < current_a.relative_path: yield (None, current_b) current_b = next(iter_b, None) else: assert current_a.relative_path == current_b.relative_path yield (current_a, current_b) current_a = next(iter_a, None) current_b = next(iter_b, None) reporter.close() @dataclass(frozen=True) class AbstractScanResult(metaclass=ABCMeta): """ Some attributes of files which are meaningful for monitoring and troubleshooting. """ @abstractclassmethod def from_files(cls, *files: Optional[AbstractPath]) -> 'AbstractScanResult': pass @dataclass class AbstractScanReport(metaclass=ABCMeta): """ Aggregation of valuable information about files after scanning. """ SCAN_RESULT_CLASS: ClassVar[Type] = AbstractScanResult @abstractmethod def add(self, *files: Optional[AbstractPath]) -> None: pass @dataclass class CountAndSampleScanReport(AbstractScanReport): """ Scan report which groups and counts files by their `AbstractScanResult` and also stores first and last seen examples of such files. """ counter_by_status: Counter = field(default_factory=Counter) samples_by_status_first: Dict[AbstractScanResult, Tuple[FileVersion, ...]] = field( default_factory=dict ) samples_by_status_last: Dict[AbstractScanResult, Tuple[FileVersion, ...]] = field( default_factory=dict ) def add(self, *files: Optional[AbstractPath]) -> None: status = self.SCAN_RESULT_CLASS.from_files(*files) self.counter_by_status[status] += 1 sample = tuple(file and file.selected_version for file in files) self.samples_by_status_first.setdefault(status, sample) self.samples_by_status_last[status] = sample b2-sdk-python-1.17.3/b2sdk/session.py000066400000000000000000000476201426424117700172670ustar00rootroot00000000000000###################################################################### # # File: b2sdk/session.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from functools import partial from enum import Enum, unique from typing import Any, Dict, Optional import logging from b2sdk.account_info.abstract import AbstractAccountInfo from b2sdk.account_info.sqlite_account_info import SqliteAccountInfo from b2sdk.account_info.exception import MissingAccountData from b2sdk.b2http import B2Http from b2sdk.cache import AbstractCache, AuthInfoCache, DummyCache from b2sdk.encryption.setting import EncryptionSetting from b2sdk.replication.setting import ReplicationConfiguration from b2sdk.exception import (InvalidAuthToken, Unauthorized) from b2sdk.file_lock import BucketRetentionSetting, FileRetentionSetting, LegalHold from b2sdk.raw_api import ALL_CAPABILITIES, REALM_URLS from b2sdk.api_config import B2HttpApiConfig, DEFAULT_HTTP_API_CONFIG logger = logging.getLogger(__name__) @unique class TokenType(Enum): API = 'api' API_TOKEN_ONLY = 'api_token_only' UPLOAD_PART = 'upload_part' UPLOAD_SMALL = 'upload_small' class B2Session: """ A facade that supplies the correct api_url and account_auth_token to methods of underlying raw_api and reauthorizes if necessary. """ SQLITE_ACCOUNT_INFO_CLASS = staticmethod(SqliteAccountInfo) B2HTTP_CLASS = staticmethod(B2Http) def __init__( self, account_info: Optional[AbstractAccountInfo] = None, cache: Optional[AbstractCache] = None, api_config: B2HttpApiConfig = DEFAULT_HTTP_API_CONFIG ): """ Initialize Session using given account info. :param account_info: an instance of :class:`~b2sdk.v2.UrlPoolAccountInfo`, or any custom class derived from :class:`~b2sdk.v2.AbstractAccountInfo` To learn more about Account Info objects, see here :class:`~b2sdk.v2.SqliteAccountInfo` :param cache: an instance of the one of the following classes: :class:`~b2sdk.cache.DummyCache`, :class:`~b2sdk.cache.InMemoryCache`, :class:`~b2sdk.cache.AuthInfoCache`, or any custom class derived from :class:`~b2sdk.cache.AbstractCache` It is used by B2Api to cache the mapping between bucket name and bucket ids. default is :class:`~b2sdk.cache.DummyCache` :param api_config """ self.raw_api = api_config.raw_api_class(self.B2HTTP_CLASS(api_config)) if account_info is None: account_info = self.SQLITE_ACCOUNT_INFO_CLASS() if cache is None: cache = AuthInfoCache(account_info) if cache is None: cache = DummyCache() self.account_info = account_info self.cache = cache self._token_callbacks = { TokenType.API: self._api_token_callback, TokenType.API_TOKEN_ONLY: self._api_token_only_callback, TokenType.UPLOAD_SMALL: self._upload_small, TokenType.UPLOAD_PART: self._upload_part, } def authorize_automatically(self): """ Perform automatic account authorization, retrieving all account data from account info object passed during initialization. """ try: self.authorize_account( self.account_info.get_realm(), self.account_info.get_application_key_id(), self.account_info.get_application_key(), ) except MissingAccountData: return False return True def authorize_account(self, realm, application_key_id, application_key): """ Perform account authorization. :param str realm: a realm to authorize account in (usually just "production") :param str application_key_id: :term:`application key ID` :param str application_key: user's :term:`application key` """ # Authorize realm_url = REALM_URLS.get(realm, realm) response = self.raw_api.authorize_account(realm_url, application_key_id, application_key) account_id = response['accountId'] allowed = response['allowed'] # Clear the cache if new account has been used if not self.account_info.is_same_account(account_id, realm): self.cache.clear() # Store the auth data self.account_info.set_auth_data( account_id=account_id, auth_token=response['authorizationToken'], api_url=response['apiUrl'], download_url=response['downloadUrl'], absolute_minimum_part_size=response['absoluteMinimumPartSize'], recommended_part_size=response['recommendedPartSize'], application_key=application_key, realm=realm, s3_api_url=response['s3ApiUrl'], allowed=allowed, application_key_id=application_key_id ) def cancel_large_file(self, file_id): return self._wrap_default_token(self.raw_api.cancel_large_file, file_id) def create_bucket( self, account_id, bucket_name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules=None, default_server_side_encryption=None, is_file_lock_enabled: Optional[bool] = None, replication: Optional[ReplicationConfiguration] = None, ): return self._wrap_default_token( self.raw_api.create_bucket, account_id, bucket_name, bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, default_server_side_encryption=default_server_side_encryption, is_file_lock_enabled=is_file_lock_enabled, replication=replication, ) def create_key( self, account_id, capabilities, key_name, valid_duration_seconds, bucket_id, name_prefix ): return self._wrap_default_token( self.raw_api.create_key, account_id, capabilities, key_name, valid_duration_seconds, bucket_id, name_prefix, ) def delete_key(self, application_key_id): return self._wrap_default_token(self.raw_api.delete_key, application_key_id) def delete_bucket(self, account_id, bucket_id): return self._wrap_default_token(self.raw_api.delete_bucket, account_id, bucket_id) def delete_file_version(self, file_id, file_name): return self._wrap_default_token(self.raw_api.delete_file_version, file_id, file_name) def download_file_from_url( self, url, range_=None, encryption: Optional[EncryptionSetting] = None ): return self._wrap_token( self.raw_api.download_file_from_url, TokenType.API_TOKEN_ONLY, url, range_=range_, encryption=encryption, ) def finish_large_file(self, file_id, part_sha1_array): return self._wrap_default_token(self.raw_api.finish_large_file, file_id, part_sha1_array) def get_download_authorization(self, bucket_id, file_name_prefix, valid_duration_in_seconds): return self._wrap_default_token( self.raw_api.get_download_authorization, bucket_id, file_name_prefix, valid_duration_in_seconds ) def get_file_info_by_id(self, file_id: str) -> Dict[str, Any]: return self._wrap_default_token(self.raw_api.get_file_info_by_id, file_id) def get_file_info_by_name(self, bucket_name: str, file_name: str) -> Dict[str, Any]: return self._wrap_default_token(self.raw_api.get_file_info_by_name, bucket_name, file_name) def get_upload_url(self, bucket_id): return self._wrap_default_token(self.raw_api.get_upload_url, bucket_id) def get_upload_part_url(self, file_id): return self._wrap_default_token(self.raw_api.get_upload_part_url, file_id) def hide_file(self, bucket_id, file_name): return self._wrap_default_token(self.raw_api.hide_file, bucket_id, file_name) def list_buckets(self, account_id, bucket_id=None, bucket_name=None): return self._wrap_default_token( self.raw_api.list_buckets, account_id, bucket_id=bucket_id, bucket_name=bucket_name, ) def list_file_names( self, bucket_id, start_file_name=None, max_file_count=None, prefix=None, ): return self._wrap_default_token( self.raw_api.list_file_names, bucket_id, start_file_name=start_file_name, max_file_count=max_file_count, prefix=prefix, ) def list_file_versions( self, bucket_id, start_file_name=None, start_file_id=None, max_file_count=None, prefix=None, ): return self._wrap_default_token( self.raw_api.list_file_versions, bucket_id, start_file_name=start_file_name, start_file_id=start_file_id, max_file_count=max_file_count, prefix=prefix, ) def list_keys(self, account_id, max_key_count=None, start_application_key_id=None): return self._wrap_default_token( self.raw_api.list_keys, account_id, max_key_count=max_key_count, start_application_key_id=start_application_key_id, ) def list_parts(self, file_id, start_part_number, max_part_count): return self._wrap_default_token( self.raw_api.list_parts, file_id, start_part_number, max_part_count ) def list_unfinished_large_files( self, bucket_id, start_file_id=None, max_file_count=None, prefix=None, ): return self._wrap_default_token( self.raw_api.list_unfinished_large_files, bucket_id, start_file_id=start_file_id, max_file_count=max_file_count, prefix=prefix, ) def start_large_file( self, bucket_id, file_name, content_type, file_info, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): return self._wrap_default_token( self.raw_api.start_large_file, bucket_id, file_name, content_type, file_info, server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, ) def update_bucket( self, account_id, bucket_id, bucket_type=None, bucket_info=None, cors_rules=None, lifecycle_rules=None, if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, default_retention: Optional[BucketRetentionSetting] = None, replication: Optional[ReplicationConfiguration] = None, ): return self._wrap_default_token( self.raw_api.update_bucket, account_id, bucket_id, bucket_type=bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, replication=replication, ) def upload_file( self, bucket_id, file_name, content_length, content_type, content_sha1, file_infos, data_stream, server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): return self._wrap_token( self.raw_api.upload_file, TokenType.UPLOAD_SMALL, bucket_id, file_name, content_length, content_type, content_sha1, file_infos, data_stream, server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, ) def upload_part( self, file_id, part_number, content_length, sha1_sum, input_stream, server_side_encryption: Optional[EncryptionSetting] = None, ): return self._wrap_token( self.raw_api.upload_part, TokenType.UPLOAD_PART, file_id, part_number, content_length, sha1_sum, input_stream, server_side_encryption, ) def get_download_url_by_id(self, file_id): return self.raw_api.get_download_url_by_id(self.account_info.get_download_url(), file_id) def get_download_url_by_name(self, bucket_name, file_name): return self.raw_api.get_download_url_by_name( self.account_info.get_download_url(), bucket_name, file_name ) def copy_file( self, source_file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_bucket_id=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): return self._wrap_default_token( self.raw_api.copy_file, source_file_id, new_file_name, bytes_range=bytes_range, metadata_directive=metadata_directive, content_type=content_type, file_info=file_info, destination_bucket_id=destination_bucket_id, destination_server_side_encryption=destination_server_side_encryption, source_server_side_encryption=source_server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, ) def copy_part( self, source_file_id, large_file_id, part_number, bytes_range=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, ): return self._wrap_default_token( self.raw_api.copy_part, source_file_id, large_file_id, part_number, bytes_range=bytes_range, destination_server_side_encryption=destination_server_side_encryption, source_server_side_encryption=source_server_side_encryption, ) def _wrap_default_token(self, raw_api_method, *args, **kwargs): return self._wrap_token(raw_api_method, TokenType.API, *args, **kwargs) def _wrap_token(self, raw_api_method, token_type, *args, **kwargs): callback = self._token_callbacks[token_type] partial_callback = partial(callback, raw_api_method, *args, **kwargs) return self._reauthorization_loop(partial_callback) def _api_token_callback(self, raw_api_method, *args, **kwargs): api_url = self.account_info.get_api_url() account_auth_token = self.account_info.get_account_auth_token() return raw_api_method(api_url, account_auth_token, *args, **kwargs) def _api_token_only_callback(self, raw_api_method, *args, **kwargs): account_auth_token = self.account_info.get_account_auth_token() return raw_api_method(account_auth_token, *args, **kwargs) def _reauthorization_loop(self, callback): auth_failure_encountered = False while 1: try: return callback() except InvalidAuthToken: if not auth_failure_encountered: auth_failure_encountered = True reauthorization_success = self.authorize_automatically() if reauthorization_success: continue raise except Unauthorized as e: raise self._add_app_key_info_to_unauthorized(e) def _add_app_key_info_to_unauthorized(self, unauthorized): """ Take an Unauthorized error and adds information from the application key about why it might have failed. """ # What's allowed? allowed = self.account_info.get_allowed() capabilities = allowed['capabilities'] bucket_name = allowed['bucketName'] name_prefix = allowed['namePrefix'] # Make a list of messages about the application key restrictions key_messages = [] if set(capabilities) != set(ALL_CAPABILITIES): key_messages.append("with capabilities '" + ','.join(capabilities) + "'") if bucket_name is not None: key_messages.append("restricted to bucket '" + bucket_name + "'") if name_prefix is not None: key_messages.append("restricted to files that start with '" + name_prefix + "'") if not key_messages: key_messages.append('with no restrictions') # Make a new message new_message = unauthorized.message if new_message == '': new_message = 'unauthorized' new_message += ' for application key ' + ', '.join(key_messages) return Unauthorized(new_message, unauthorized.code) def _get_upload_data(self, bucket_id): """ Take ownership of an upload URL / auth token for the bucket and return it. """ account_info = self.account_info upload_url, upload_auth_token = account_info.take_bucket_upload_url(bucket_id) if None not in (upload_url, upload_auth_token): return upload_url, upload_auth_token response = self.get_upload_url(bucket_id) return response['uploadUrl'], response['authorizationToken'] def _get_upload_part_data(self, file_id): """ Make sure that we have an upload URL and auth token for the given bucket and return it. """ account_info = self.account_info upload_url, upload_auth_token = account_info.take_large_file_upload_url(file_id) if None not in (upload_url, upload_auth_token): return upload_url, upload_auth_token response = self.get_upload_part_url(file_id) return response['uploadUrl'], response['authorizationToken'] def _upload_small(self, f, bucket_id, *args, **kwargs): upload_url, upload_auth_token = self._get_upload_data(bucket_id) response = f(upload_url, upload_auth_token, *args, **kwargs) self.account_info.put_bucket_upload_url(bucket_id, upload_url, upload_auth_token) return response def _upload_part(self, f, file_id, *args, **kwargs): upload_url, upload_auth_token = self._get_upload_part_data(file_id) response = f(upload_url, upload_auth_token, *args, **kwargs) self.account_info.put_large_file_upload_url(file_id, upload_url, upload_auth_token) return response def update_file_retention( self, file_id, file_name, file_retention: FileRetentionSetting, bypass_governance: bool = False, ): return self._wrap_default_token( self.raw_api.update_file_retention, file_id, file_name, file_retention, bypass_governance, ) def update_file_legal_hold( self, file_id, file_name, legal_hold: LegalHold, ): return self._wrap_default_token( self.raw_api.update_file_legal_hold, file_id, file_name, legal_hold, ) b2-sdk-python-1.17.3/b2sdk/stream/000077500000000000000000000000001426424117700165145ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/stream/__init__.py000066400000000000000000000010721426424117700206250ustar00rootroot00000000000000###################################################################### # # File: b2sdk/stream/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from .hashing import StreamWithHash from .progress import ReadingStreamWithProgress, WritingStreamWithProgress from .range import RangeOfInputStream __all__ = [ 'RangeOfInputStream', 'ReadingStreamWithProgress', 'StreamWithHash', 'WritingStreamWithProgress', ] b2-sdk-python-1.17.3/b2sdk/stream/base.py000066400000000000000000000007461426424117700200070ustar00rootroot00000000000000###################################################################### # # File: b2sdk/stream/base.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import io class ReadOnlyStreamMixin: def writeable(self): return False def write(self, data): raise io.UnsupportedOperation('Cannot accept a write to a read-only stream') b2-sdk-python-1.17.3/b2sdk/stream/chained.py000066400000000000000000000122151426424117700204620ustar00rootroot00000000000000###################################################################### # # File: b2sdk/stream/chained.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import io from abc import ABCMeta, abstractmethod from b2sdk.stream.base import ReadOnlyStreamMixin class ChainedStream(ReadOnlyStreamMixin, io.IOBase): """ Chains multiple streams in single stream, sort of what :py:class:`itertools.chain` does for iterators. Cleans up buffers of underlying streams when closed. Can be seeked to beginning (when retrying upload, for example). Closes underlying streams as soon as they reaches EOF, but clears their buffers when the chained stream is closed for underlying streams that follow :py:class:`b2sdk.v2.StreamOpener` cleanup interface, for example :py:class:`b2sdk.v2.CachedBytesStreamOpener` """ def __init__(self, stream_openers): """ :param list stream_openeres: list of callables that return opened streams """ stream_openers = list(stream_openers) if not stream_openers: raise ValueError('chain_links cannot be empty') self.stream_openers = stream_openers self._stream_openers_iterator = iter(self.stream_openers) self._current_stream = None self._pos = 0 super(ChainedStream, self).__init__() @property def stream(self): """ Return currently processed stream. """ if self._current_stream is None: self._next_stream() return self._current_stream def _reset_chain(self): if self._current_stream is not None: self._current_stream.close() self._current_stream = None self._stream_openers_iterator = iter(self.stream_openers) self._pos = 0 def _next_stream(self): next_stream_opener = next(self._stream_openers_iterator, None) if next_stream_opener is not None: if self._current_stream is not None: self._current_stream.close() self._current_stream = next_stream_opener() def seekable(self): return True def tell(self): return self._pos def seek(self, pos, whence=0): """ Resets stream to the beginning. :param int pos: only allowed value is ``0`` :param int whence: only allowed value is ``0`` """ if pos != 0 or whence != 0: raise io.UnsupportedOperation('Chained stream can only be seeked to beginning') self._reset_chain() return self.tell() def readable(self): return True def read(self, size=None): """ Read at most `size` bytes from underlying streams, or all available data, if `size` is None or negative. Open the streams only when their data is needed, and possibly leave them open and part-way read for further reading - by subsequent calls to this method. :param int,None size: number of bytes to read. If omitted, ``None``, or negative data is read and returned until EOF from final stream is reached :return: data read from the stream """ byte_arrays = [] if size < 0 or size is None: while 1: current_stream = self.stream buff = current_stream.read() byte_arrays.append(buff) if not buff: self._next_stream() if self.stream is current_stream: break else: remaining = size while 1: current_stream = self.stream buff = current_stream.read(remaining) byte_arrays.append(buff) remaining -= len(buff) if remaining == 0: # no need to open any other streams - we're satisfied break if not buff: self._next_stream() if self.stream is current_stream: break if not byte_arrays: data = byte_arrays[0] else: data = bytes().join(byte_arrays) self._pos += len(data) return data def close(self): if self._current_stream is not None: self._current_stream.close() for stream_opener in self.stream_openers: if hasattr(stream_opener, 'cleanup'): stream_opener.cleanup() super(ChainedStream, self).close() class StreamOpener(metaclass=ABCMeta): """ Abstract class to define stream opener with cleanup. """ @abstractmethod def __call__(self): """ Create or open the stream to read and return. Can be called multiple times, but streamed data may be cached and reused. """ def cleanup(self): """ Clean up stream opener after chained stream closes. Can be used for cleaning cached data that are stored in memory to allow resetting chained stream without getting this data more than once, eg. data downloaded from external source. """ b2-sdk-python-1.17.3/b2sdk/stream/hashing.py000066400000000000000000000045201426424117700205100ustar00rootroot00000000000000###################################################################### # # File: b2sdk/stream/hashing.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import hashlib import io from b2sdk.stream.wrapper import StreamWithLengthWrapper from b2sdk.stream.base import ReadOnlyStreamMixin class StreamWithHash(ReadOnlyStreamMixin, StreamWithLengthWrapper): """ Wrap a file-like object, calculates SHA1 while reading and appends hash at the end. """ def __init__(self, stream, stream_length=None): """ :param stream: the stream to read from """ self.digest = self.get_digest() total_length = None if stream_length is not None: total_length = stream_length + self.digest.digest_size * 2 super(StreamWithHash, self).__init__(stream, length=total_length) self.hash = None self.hash_read = 0 def seek(self, pos, whence=0): """ Seek to a given position in the stream. :param int pos: position in the stream """ if pos != 0 or whence != 0: raise io.UnsupportedOperation('Stream with hash can only be seeked to beginning') self.digest = self.get_digest() self.hash = None self.hash_read = 0 return super(StreamWithHash, self).seek(0) def read(self, size=None): """ Read data from the stream. :param int size: number of bytes to read :return: read data :rtype: bytes|None """ data = b'' if self.hash is None: data = super(StreamWithHash, self).read(size) # Update hash self.digest.update(data) # Check for end of stream if size is None or len(data) < size: self.hash = self.digest.hexdigest() if size is not None: size -= len(data) if self.hash is not None: # The end of stream was reached, return hash now size = size or len(self.hash) data += str.encode(self.hash[self.hash_read:self.hash_read + size]) self.hash_read += size return data @classmethod def get_digest(cls): return hashlib.sha1() b2-sdk-python-1.17.3/b2sdk/stream/progress.py000066400000000000000000000061111426424117700207310ustar00rootroot00000000000000###################################################################### # # File: b2sdk/stream/progress.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.stream.wrapper import StreamWrapper class AbstractStreamWithProgress(StreamWrapper): """ Wrap a file-like object and updates a ProgressListener as data is read / written. In the abstract class, read and write methods do not update the progress - child classes shall do it. """ def __init__(self, stream, progress_listener, offset=0): """ :param stream: the stream to read from or write to :param b2sdk.v2.AbstractProgressListener progress_listener: the listener that we tell about progress :param int offset: the starting byte offset in the file """ super(AbstractStreamWithProgress, self).__init__(stream) assert progress_listener is not None self.progress_listener = progress_listener self.bytes_completed = 0 self.offset = offset def _progress_update(self, delta): self.bytes_completed += delta self.progress_listener.bytes_completed(self.bytes_completed + self.offset) class ReadingStreamWithProgress(AbstractStreamWithProgress): """ Wrap a file-like object, updates progress while reading. """ def __init__(self, *args, **kwargs): length = kwargs.pop('length', None) super(ReadingStreamWithProgress, self).__init__(*args, **kwargs) self.length = length def read(self, size=None): """ Read data from the stream. :param int size: number of bytes to read :return: data read from the stream """ data = super(ReadingStreamWithProgress, self).read(size) self._progress_update(len(data)) return data def seek(self, pos, whence=0): pos = super(ReadingStreamWithProgress, self).seek(pos, whence=whence) # reset progress to current stream position - assumption is that ReadingStreamWithProgress would not be used # for random access streams, and seek is only used to reset stream to beginning to retry file upload # and multipart file upload would open and use different file descriptor for each part; # this logic cannot be used for WritingStreamWithProgress because multipart download has to use # single file descriptor and synchronize writes so seeking cannot be understood there as progress reset # and writing progress is always monotonic self.bytes_completed = pos return pos def __len__(self): return self.length class WritingStreamWithProgress(AbstractStreamWithProgress): """ Wrap a file-like object; updates progress while writing. """ def write(self, data): """ Write data to the stream. :param bytes data: data to write to the stream """ self._progress_update(len(data)) return super(WritingStreamWithProgress, self).write(data) b2-sdk-python-1.17.3/b2sdk/stream/range.py000066400000000000000000000050241426424117700201630ustar00rootroot00000000000000###################################################################### # # File: b2sdk/stream/range.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import io from b2sdk.stream.wrapper import StreamWithLengthWrapper from b2sdk.stream.base import ReadOnlyStreamMixin class RangeOfInputStream(ReadOnlyStreamMixin, StreamWithLengthWrapper): """ Wrap a file-like object (read only) and read the selected range of the file. """ def __init__(self, stream, offset, length): """ :param stream: a seekable stream :param int offset: offset in the stream :param int length: max number of bytes to read """ super(RangeOfInputStream, self).__init__(stream, length) self.offset = offset self.relative_pos = 0 self.stream.seek(self.offset) def seek(self, pos, whence=0): """ Seek to a given position in the stream. :param int pos: position in the stream relative to steam offset :return: new position relative to stream offset :rtype: int """ if whence != 0: raise io.UnsupportedOperation('only SEEK_SET is supported') abs_pos = super(RangeOfInputStream, self).seek(self.offset + pos) self.relative_pos = abs_pos - self.offset return self.tell() def tell(self): """ Return current stream position relative to offset. :rtype: int """ return self.relative_pos def read(self, size=None): """ Read data from the stream. :param int size: number of bytes to read :return: data read from the stream :rtype: bytes """ remaining = max(0, self.length - self.relative_pos) if size is None: to_read = remaining else: to_read = min(size, remaining) data = self.stream.read(to_read) self.relative_pos += len(data) return data def close(self): super().close() # TODO: change the use cases of this class to close the file objects passed to it, instead of having # RangeOfInputStream close it's members upon garbage collection self.stream.close() def wrap_with_range(stream, stream_length, range_offset, range_length): if range_offset == 0 and range_length == stream_length: return stream return RangeOfInputStream(stream, range_offset, range_length) b2-sdk-python-1.17.3/b2sdk/stream/wrapper.py000066400000000000000000000043511426424117700205510ustar00rootroot00000000000000###################################################################### # # File: b2sdk/stream/wrapper.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import io class StreamWrapper(io.IOBase): """ Wrapper for a file-like object. """ def __init__(self, stream): """ :param stream: the stream to read from or write to """ self.stream = stream super(StreamWrapper, self).__init__() def seekable(self): return self.stream.seekable() def seek(self, pos, whence=0): """ Seek to a given position in the stream. :param int pos: position in the stream :return: new absolute position :rtype: int """ return self.stream.seek(pos, whence) def tell(self): """ Return current stream position. :rtype: int """ return self.stream.tell() def truncate(self, size=None): return self.stream.truncate(size) def flush(self): """ Flush the stream. """ self.stream.flush() def readable(self): return self.stream.readable() def read(self, size=None): """ Read data from the stream. :param int size: number of bytes to read :return: data read from the stream """ if size is not None: return self.stream.read(size) else: return self.stream.read() def writable(self): return self.stream.writable() def write(self, data): """ Write data to the stream. :param data: a data to write to the stream """ return self.stream.write(data) class StreamWithLengthWrapper(StreamWrapper): """ Wrapper for a file-like object that supports `__len__` interface """ def __init__(self, stream, length=None): """ :param stream: the stream to read from or write to :param int length: length of the stream """ super(StreamWithLengthWrapper, self).__init__(stream) self.length = length def __len__(self): return self.length b2-sdk-python-1.17.3/b2sdk/sync/000077500000000000000000000000001426424117700161755ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/sync/__init__.py000066400000000000000000000004411426424117700203050ustar00rootroot00000000000000###################################################################### # # File: b2sdk/sync/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/sync/action.py000066400000000000000000000354241426424117700200340ustar00rootroot00000000000000###################################################################### # # File: b2sdk/sync/action.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging import os from abc import ABCMeta, abstractmethod from ..bucket import Bucket from ..http_constants import SRC_LAST_MODIFIED_MILLIS from ..scan.path import B2Path from ..transfer.outbound.upload_source import UploadSourceLocalFile from .encryption_provider import AbstractSyncEncryptionSettingsProvider from .report import SyncFileReporter logger = logging.getLogger(__name__) class AbstractAction(metaclass=ABCMeta): """ An action to take, such as uploading, downloading, or deleting a file. Multi-threaded tasks create a sequence of Actions which are then run by a pool of threads. An action can depend on other actions completing. An example of this is making sure a CreateBucketAction happens before an UploadFileAction. """ def run(self, bucket, reporter, dry_run=False): """ Main action routine. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors :param dry_run: if True, perform a dry run :type dry_run: bool """ try: if not dry_run: self.do_action(bucket, reporter) self.do_report(bucket, reporter) except Exception as e: logger.exception('an exception occurred in a sync action') reporter.error(str(self) + ": " + repr(e) + ' ' + str(e)) raise # Re-throw so we can identify failed actions @abstractmethod def get_bytes(self): """ Return the number of bytes to transfer for this action. :rtype: int """ @abstractmethod def do_action(self, bucket, reporter): """ Perform the action, returning only after the action is completed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ @abstractmethod def do_report(self, bucket, reporter): """ Report the action performed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ class B2UploadAction(AbstractAction): """ File uploading action. """ def __init__( self, local_full_path, relative_name, b2_file_name, mod_time_millis, size, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider, ): """ :param str local_full_path: a local file path :param str relative_name: a relative file name :param str b2_file_name: a name of a new remote file :param int mod_time_millis: file modification time in milliseconds :param int size: a file size :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ self.local_full_path = local_full_path self.relative_name = relative_name self.b2_file_name = b2_file_name self.mod_time_millis = mod_time_millis self.size = size self.encryption_settings_provider = encryption_settings_provider def get_bytes(self): """ Return file size. :rtype: int """ return self.size def do_action(self, bucket, reporter): """ Perform the uploading action, returning only after the action is completed. :param b2sdk.v2.Bucket bucket: a Bucket object :param reporter: a place to report errors """ if reporter: progress_listener = SyncFileReporter(reporter) else: progress_listener = None file_info = {SRC_LAST_MODIFIED_MILLIS: str(self.mod_time_millis)} encryption = self.encryption_settings_provider.get_setting_for_upload( bucket=bucket, b2_file_name=self.b2_file_name, file_info=file_info, length=self.size, ) bucket.upload( UploadSourceLocalFile(self.local_full_path), self.b2_file_name, file_info=file_info, progress_listener=progress_listener, encryption=encryption, ) def do_report(self, bucket, reporter): """ Report the uploading action performed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ reporter.print_completion('upload ' + self.relative_name) def __str__(self): return 'b2_upload(%s, %s, %s)' % ( self.local_full_path, self.b2_file_name, self.mod_time_millis ) class B2HideAction(AbstractAction): def __init__(self, relative_name, b2_file_name): """ :param relative_name: a relative file name :type relative_name: str :param b2_file_name: a name of a remote file :type b2_file_name: str """ self.relative_name = relative_name self.b2_file_name = b2_file_name def get_bytes(self): """ Return file size. :return: always zero :rtype: int """ return 0 def do_action(self, bucket, reporter): """ Perform the hiding action, returning only after the action is completed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ bucket.hide_file(self.b2_file_name) def do_report(self, bucket, reporter): """ Report the hiding action performed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ reporter.update_transfer(1, 0) reporter.print_completion('hide ' + self.relative_name) def __str__(self): return 'b2_hide(%s)' % (self.b2_file_name,) class B2DownloadAction(AbstractAction): def __init__( self, source_path: B2Path, b2_file_name: str, local_full_path: str, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider, ): """ :param b2sdk.v2.B2Path source_path: the file to be downloaded :param str b2_file_name: b2_file_name :param str local_full_path: a local file path :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ self.source_path = source_path self.b2_file_name = b2_file_name self.local_full_path = local_full_path self.encryption_settings_provider = encryption_settings_provider def get_bytes(self): """ Return file size. :rtype: int """ return self.source_path.size def _ensure_directory_existence(self): parent_dir = os.path.dirname(self.local_full_path) if not os.path.isdir(parent_dir): try: os.makedirs(parent_dir) except OSError: pass if not os.path.isdir(parent_dir): raise Exception('could not create directory %s' % (parent_dir,)) def do_action(self, bucket, reporter): """ Perform the downloading action, returning only after the action is completed. :param b2sdk.v2.Bucket bucket: a Bucket object :param reporter: a place to report errors """ self._ensure_directory_existence() if reporter: progress_listener = SyncFileReporter(reporter) else: progress_listener = None # Download the file to a .tmp file download_path = self.local_full_path + '.b2.sync.tmp' encryption = self.encryption_settings_provider.get_setting_for_download( bucket=bucket, file_version=self.source_path.selected_version, ) downloaded_file = bucket.download_file_by_id( self.source_path.selected_version.id_, progress_listener=progress_listener, encryption=encryption, ) downloaded_file.save_to(download_path) # Move the file into place try: os.unlink(self.local_full_path) except OSError: pass os.rename(download_path, self.local_full_path) def do_report(self, bucket, reporter): """ Report the downloading action performed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ reporter.print_completion('dnload ' + self.source_path.relative_path) def __str__(self): return ( 'b2_download(%s, %s, %s, %d)' % ( self.b2_file_name, self.source_path.selected_version.id_, self.local_full_path, self.source_path.mod_time ) ) class B2CopyAction(AbstractAction): """ File copying action. """ def __init__( self, b2_file_name: str, source_path: B2Path, dest_b2_file_name, source_bucket: Bucket, destination_bucket: Bucket, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider, ): """ :param str b2_file_name: a b2_file_name :param b2sdk.v2.B2Path source_path: the file to be copied :param str dest_b2_file_name: a name of a destination remote file :param Bucket source_bucket: bucket to copy from :param Bucket destination_bucket: bucket to copy to :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ self.b2_file_name = b2_file_name self.source_path = source_path self.dest_b2_file_name = dest_b2_file_name self.encryption_settings_provider = encryption_settings_provider self.source_bucket = source_bucket self.destination_bucket = destination_bucket def get_bytes(self): """ Return file size. :rtype: int """ return self.source_path.size def do_action(self, bucket, reporter): """ Perform the copying action, returning only after the action is completed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ if reporter: progress_listener = SyncFileReporter(reporter) else: progress_listener = None source_encryption = self.encryption_settings_provider.get_source_setting_for_copy( bucket=self.source_bucket, source_file_version=self.source_path.selected_version, ) destination_encryption = self.encryption_settings_provider.get_destination_setting_for_copy( bucket=self.destination_bucket, source_file_version=self.source_path.selected_version, dest_b2_file_name=self.dest_b2_file_name, ) bucket.copy( self.source_path.selected_version.id_, self.dest_b2_file_name, length=self.source_path.size, progress_listener=progress_listener, destination_encryption=destination_encryption, source_encryption=source_encryption, source_file_info=self.source_path.selected_version.file_info, source_content_type=self.source_path.selected_version.content_type, ) def do_report(self, bucket, reporter): """ Report the copying action performed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ reporter.print_completion('copy ' + self.source_path.relative_path) def __str__(self): return ( 'b2_copy(%s, %s, %s, %d)' % ( self.b2_file_name, self.source_path.selected_version.id_, self.dest_b2_file_name, self.source_path.mod_time ) ) class B2DeleteAction(AbstractAction): def __init__(self, relative_name, b2_file_name, file_id, note): """ :param str relative_name: a relative file name :param str b2_file_name: a name of a remote file :param str file_id: a file ID :param str note: a deletion note """ self.relative_name = relative_name self.b2_file_name = b2_file_name self.file_id = file_id self.note = note def get_bytes(self): """ Return file size. :return: always zero :rtype: int """ return 0 def do_action(self, bucket, reporter): """ Perform the deleting action, returning only after the action is completed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ bucket.api.delete_file_version(self.file_id, self.b2_file_name) def do_report(self, bucket, reporter): """ Report the deleting action performed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ reporter.update_transfer(1, 0) reporter.print_completion('delete ' + self.relative_name + ' ' + self.note) def __str__(self): return 'b2_delete(%s, %s, %s)' % (self.b2_file_name, self.file_id, self.note) class LocalDeleteAction(AbstractAction): def __init__(self, relative_name, full_path): """ :param relative_name: a relative file name :type relative_name: str :param full_path: a full local path :type: str """ self.relative_name = relative_name self.full_path = full_path def get_bytes(self): """ Return file size. :return: always zero :rtype: int """ return 0 def do_action(self, bucket, reporter): """ Perform the deleting of a local file action, returning only after the action is completed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ os.unlink(self.full_path) def do_report(self, bucket, reporter): """ Report the deleting of a local file action performed. :param bucket: a Bucket object :type bucket: b2sdk.bucket.Bucket :param reporter: a place to report errors """ reporter.update_transfer(1, 0) reporter.print_completion('delete ' + self.relative_name) def __str__(self): return 'local_delete(%s)' % (self.full_path) b2-sdk-python-1.17.3/b2sdk/sync/encryption_provider.py000066400000000000000000000075771426424117700226730ustar00rootroot00000000000000###################################################################### # # File: b2sdk/sync/encryption_provider.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import ABCMeta, abstractmethod from typing import Dict, Optional from ..encryption.setting import EncryptionSetting from ..bucket import Bucket from ..file_version import FileVersion class AbstractSyncEncryptionSettingsProvider(metaclass=ABCMeta): """ Object which provides an appropriate EncryptionSetting object for sync, i.e. complex operations with multiple sources and destinations """ @abstractmethod def get_setting_for_upload( self, bucket: Bucket, b2_file_name: str, file_info: Optional[dict], length: int, ) -> Optional[EncryptionSetting]: """ Return an EncryptionSetting for uploading an object or None if server should decide. """ @abstractmethod def get_source_setting_for_copy( self, bucket: Bucket, source_file_version: FileVersion, ) -> Optional[EncryptionSetting]: """ Return an EncryptionSetting for a source of copying an object or None if not required """ @abstractmethod def get_destination_setting_for_copy( self, bucket: Bucket, dest_b2_file_name: str, source_file_version: FileVersion, target_file_info: Optional[dict] = None, ) -> Optional[EncryptionSetting]: """ Return an EncryptionSetting for a destination for copying an object or None if server should decide """ @abstractmethod def get_setting_for_download( self, bucket: Bucket, file_version: FileVersion, ) -> Optional[EncryptionSetting]: """ Return an EncryptionSetting for downloading an object from, or None if not required """ class ServerDefaultSyncEncryptionSettingsProvider(AbstractSyncEncryptionSettingsProvider): """ Encryption settings provider which assumes setting-less reads and a bucket default for writes. """ def get_setting_for_upload(self, *args, **kwargs) -> None: return None def get_source_setting_for_copy(self, *args, **kwargs) -> None: return None def get_destination_setting_for_copy(self, *args, **kwargs) -> None: return None def get_setting_for_download(self, *args, **kwargs) -> None: return None SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER = ServerDefaultSyncEncryptionSettingsProvider() class BasicSyncEncryptionSettingsProvider(AbstractSyncEncryptionSettingsProvider): """ Basic encryption setting provider that supports exactly one encryption setting per bucket for reading and one encryption setting per bucket for writing """ def __init__( self, read_bucket_settings: Dict[str, Optional[EncryptionSetting]], write_bucket_settings: Dict[str, Optional[EncryptionSetting]], ): self.read_bucket_settings = read_bucket_settings self.write_bucket_settings = write_bucket_settings def get_setting_for_upload(self, bucket, *args, **kwargs) -> Optional[EncryptionSetting]: return self.write_bucket_settings.get(bucket.name) def get_source_setting_for_copy(self, bucket, *args, **kwargs) -> None: return self.read_bucket_settings.get(bucket.name) def get_destination_setting_for_copy(self, bucket, *args, **kwargs) -> Optional[EncryptionSetting]: return self.write_bucket_settings.get(bucket.name) def get_setting_for_download(self, bucket, *args, **kwargs) -> None: return self.read_bucket_settings.get(bucket.name) def __repr__(self): return '<%s:%s>' % (self.__class__.__name__, self.bucket_settings) b2-sdk-python-1.17.3/b2sdk/sync/exception.py000066400000000000000000000006511426424117700205470ustar00rootroot00000000000000###################################################################### # # File: b2sdk/sync/exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from ..exception import B2SimpleError from ..scan.exception import BaseDirectoryError class IncompleteSync(B2SimpleError): pass b2-sdk-python-1.17.3/b2sdk/sync/policy.py000066400000000000000000000421071426424117700200520ustar00rootroot00000000000000###################################################################### # # File: b2sdk/sync/policy.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging from abc import ABCMeta, abstractmethod from enum import Enum, unique from typing import Optional from ..exception import DestFileNewer from ..scan.exception import InvalidArgument from ..scan.folder import AbstractFolder from ..scan.path import AbstractPath from .action import B2CopyAction, B2DeleteAction, B2DownloadAction, B2HideAction, B2UploadAction, LocalDeleteAction from .encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, AbstractSyncEncryptionSettingsProvider ONE_DAY_IN_MS = 24 * 60 * 60 * 1000 logger = logging.getLogger(__name__) @unique class NewerFileSyncMode(Enum): """ Mode of handling files newer on destination than on source """ SKIP = 101 #: skip syncing such file REPLACE = 102 #: replace the file on the destination with the (older) file on source RAISE_ERROR = 103 #: raise a non-transient error, failing the sync operation @unique class CompareVersionMode(Enum): """ Mode of comparing versions of files to determine what should be synced and what shouldn't """ MODTIME = 201 #: use file modification time on source filesystem SIZE = 202 #: compare using file size NONE = 203 #: compare using file name only class AbstractFileSyncPolicy(metaclass=ABCMeta): """ Abstract policy class. """ DESTINATION_PREFIX = NotImplemented SOURCE_PREFIX = NotImplemented def __init__( self, source_path: AbstractPath, source_folder: AbstractFolder, dest_path: AbstractPath, dest_folder: AbstractFolder, now_millis: int, keep_days: int, newer_file_mode: NewerFileSyncMode, compare_threshold: int, compare_version_mode: CompareVersionMode = CompareVersionMode.MODTIME, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ :param b2sdk.v2.AbstractPath source_path: source file object :param b2sdk.v2.AbstractFolder source_folder: source folder object :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder dest_folder: destination folder object :param int now_millis: current time in milliseconds :param int keep_days: days to keep before delete :param b2sdk.v2.NewerFileSyncMode newer_file_mode: setting which determines handling for destination files newer than on the source :param int compare_threshold: when comparing with size or time for sync :param b2sdk.v2.CompareVersionMode compare_version_mode: how to compare source and destination files :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ self._source_path = source_path self._source_folder = source_folder self._dest_path = dest_path self._keep_days = keep_days self._newer_file_mode = newer_file_mode self._compare_version_mode = compare_version_mode self._compare_threshold = compare_threshold self._dest_folder = dest_folder self._now_millis = now_millis self._transferred = False self._encryption_settings_provider = encryption_settings_provider def _should_transfer(self): """ Decide whether to transfer the file from the source to the destination. """ if self._source_path is None or not self._source_path.is_visible(): # No source file. Nothing to transfer. return False elif self._dest_path is None: # Source file exists, but no destination file. Always transfer. return True else: # Both exist. Transfer only if the two are different. return self.files_are_different( self._source_path, self._dest_path, self._compare_threshold, self._compare_version_mode, self._newer_file_mode, ) @classmethod def files_are_different( cls, source_path: AbstractPath, dest_path: AbstractPath, compare_threshold: Optional[int] = None, compare_version_mode: CompareVersionMode = CompareVersionMode.MODTIME, newer_file_mode: NewerFileSyncMode = NewerFileSyncMode.RAISE_ERROR, ): """ Compare two files and determine if the the destination file should be replaced by the source file. :param b2sdk.v2.AbstractPath source_path: source file object :param b2sdk.v2.AbstractPath dest_path: destination file object :param int compare_threshold: compare threshold when comparing by time or size :param b2sdk.v2.CompareVersionMode compare_version_mode: source file version comparator method :param b2sdk.v2.NewerFileSyncMode newer_file_mode: newer destination handling method """ # Optionally set a compare threshold for fuzzy comparison compare_threshold = compare_threshold or 0 # Compare using file name only if compare_version_mode == CompareVersionMode.NONE: return False # Compare using modification time elif compare_version_mode == CompareVersionMode.MODTIME: # Get the modification time of the latest versions source_mod_time = source_path.mod_time dest_mod_time = dest_path.mod_time diff_mod_time = abs(source_mod_time - dest_mod_time) compare_threshold_exceeded = diff_mod_time > compare_threshold logger.debug( 'File %s: source time %s, dest time %s, diff %s, threshold %s, diff > threshold %s', source_path.relative_path, source_mod_time, dest_mod_time, diff_mod_time, compare_threshold, compare_threshold_exceeded, ) if compare_threshold_exceeded: # Source is newer if dest_mod_time < source_mod_time: return True # Source is older elif source_mod_time < dest_mod_time: if newer_file_mode == NewerFileSyncMode.REPLACE: return True elif newer_file_mode == NewerFileSyncMode.SKIP: return False else: raise DestFileNewer( dest_path, source_path, cls.DESTINATION_PREFIX, cls.SOURCE_PREFIX ) # Compare using file size elif compare_version_mode == CompareVersionMode.SIZE: # Get file size of the latest versions source_size = source_path.size dest_size = dest_path.size diff_size = abs(source_size - dest_size) compare_threshold_exceeded = diff_size > compare_threshold logger.debug( 'File %s: source size %s, dest size %s, diff %s, threshold %s, diff > threshold %s', source_path.relative_path, source_size, dest_size, diff_size, compare_threshold, compare_threshold_exceeded, ) # Replace if size difference is over threshold return compare_threshold_exceeded else: raise InvalidArgument('compare_version_mode', 'is invalid option') def get_all_actions(self): """ Yield file actions. """ if self._should_transfer(): yield self._make_transfer_action() self._transferred = True assert self._dest_path is not None or self._source_path is not None for action in self._get_hide_delete_actions(): yield action def _get_hide_delete_actions(self): """ Subclass policy can override this to hide or delete files. """ return [] def _get_source_mod_time(self): return self._source_path.mod_time @abstractmethod def _make_transfer_action(self): """ Return an action representing transfer of file according to the selected policy. """ class DownPolicy(AbstractFileSyncPolicy): """ File is synced down (from the cloud to disk). """ DESTINATION_PREFIX = 'local://' SOURCE_PREFIX = 'b2://' def _make_transfer_action(self): return B2DownloadAction( self._source_path, self._source_folder.make_full_path(self._source_path.relative_path), self._dest_folder.make_full_path(self._source_path.relative_path), self._encryption_settings_provider, ) class UpPolicy(AbstractFileSyncPolicy): """ File is synced up (from disk the cloud). """ DESTINATION_PREFIX = 'b2://' SOURCE_PREFIX = 'local://' def _make_transfer_action(self): return B2UploadAction( self._source_folder.make_full_path(self._source_path.relative_path), self._source_path.relative_path, self._dest_folder.make_full_path(self._source_path.relative_path), self._get_source_mod_time(), self._source_path.size, self._encryption_settings_provider, ) class UpAndDeletePolicy(UpPolicy): """ File is synced up (from disk to the cloud) and the delete flag is SET. """ def _get_hide_delete_actions(self): for action in super(UpAndDeletePolicy, self)._get_hide_delete_actions(): yield action for action in make_b2_delete_actions( self._source_path, self._dest_path, self._dest_folder, self._transferred, ): yield action class UpAndKeepDaysPolicy(UpPolicy): """ File is synced up (from disk to the cloud) and the keepDays flag is SET. """ def _get_hide_delete_actions(self): for action in super(UpAndKeepDaysPolicy, self)._get_hide_delete_actions(): yield action for action in make_b2_keep_days_actions( self._source_path, self._dest_path, self._dest_folder, self._transferred, self._keep_days, self._now_millis, ): yield action class DownAndDeletePolicy(DownPolicy): """ File is synced down (from the cloud to disk) and the delete flag is SET. """ def _get_hide_delete_actions(self): for action in super(DownAndDeletePolicy, self)._get_hide_delete_actions(): yield action if self._dest_path is not None and ( self._source_path is None or not self._source_path.is_visible() ): yield LocalDeleteAction( self._dest_path.relative_path, self._dest_folder.make_full_path(self._dest_path.relative_path) ) class DownAndKeepDaysPolicy(DownPolicy): """ File is synced down (from the cloud to disk) and the keepDays flag is SET. """ pass class CopyPolicy(AbstractFileSyncPolicy): """ File is copied (server-side). """ DESTINATION_PREFIX = 'b2://' SOURCE_PREFIX = 'b2://' def _make_transfer_action(self): return B2CopyAction( self._source_folder.make_full_path(self._source_path.relative_path), self._source_path, self._dest_folder.make_full_path(self._source_path.relative_path), self._source_folder.bucket, self._dest_folder.bucket, self._encryption_settings_provider, ) class CopyAndDeletePolicy(CopyPolicy): """ File is copied (server-side) and the delete flag is SET. """ def _get_hide_delete_actions(self): for action in super()._get_hide_delete_actions(): yield action for action in make_b2_delete_actions( self._source_path, self._dest_path, self._dest_folder, self._transferred, ): yield action class CopyAndKeepDaysPolicy(CopyPolicy): """ File is copied (server-side) and the keepDays flag is SET. """ def _get_hide_delete_actions(self): for action in super()._get_hide_delete_actions(): yield action for action in make_b2_keep_days_actions( self._source_path, self._dest_path, self._dest_folder, self._transferred, self._keep_days, self._now_millis, ): yield action def make_b2_delete_note(version, index, transferred): """ Create a note message for delete action. :param b2sdk.v2.FileVersion version: an object which contains file version info :param int index: file version index :param bool transferred: if True, file has been transferred, False otherwise """ note = '' if version.action == 'hide': note = '(hide marker)' elif transferred or 0 < index: note = '(old version)' return note def make_b2_delete_actions( source_path: AbstractPath, dest_path: AbstractPath, dest_folder: AbstractFolder, transferred: bool, ): """ Create the actions to delete files stored on B2, which are not present locally. :param b2sdk.v2.AbstractPath source_path: source file object :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder dest_folder: destination folder :param bool transferred: if True, file has been transferred, False otherwise """ if dest_path is None: # B2 does not really store folders, so there is no need to hide # them or delete them return for version_index, version in enumerate(dest_path.all_versions): keep = (version_index == 0) and (source_path is not None) and not transferred if not keep: yield B2DeleteAction( dest_path.relative_path, dest_folder.make_full_path(dest_path.relative_path), version.id_, make_b2_delete_note(version, version_index, transferred), ) def make_b2_keep_days_actions( source_path: AbstractPath, dest_path: AbstractPath, dest_folder: AbstractFolder, transferred: bool, keep_days: int, now_millis: int, ): """ Create the actions to hide or delete existing versions of a file stored in b2. When keepDays is set, all files that were visible any time from keepDays ago until now must be kept. If versions were uploaded 5 days ago, 15 days ago, and 25 days ago, and the keepDays is 10, only the 25-day old version can be deleted. The 15 day-old version was visible 10 days ago. :param b2sdk.v2.AbstractPath source_path: source file object :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder dest_folder: destination folder object :param bool transferred: if True, file has been transferred, False otherwise :param int keep_days: how many days to keep a file :param int now_millis: current time in milliseconds """ deleting = False if dest_path is None: # B2 does not really store folders, so there is no need to hide # them or delete them return for version_index, version in enumerate(dest_path.all_versions): # How old is this version? age_days = (now_millis - version.mod_time_millis) / ONE_DAY_IN_MS # Mostly, the versions are ordered by time, newest first, # BUT NOT ALWAYS. The mod time we have is the src_last_modified_millis # from the file info (if present), or the upload start time # (if not present). The user-specified src_last_modified_millis # may not be in order. Because of that, we no longer # assert that age_days is non-decreasing. # # Note that if there is an out-of-order date that is old enough # to trigger deletions, all of the versions uploaded before that # (the ones after it in the list) will be deleted, even if they # aren't over the age threshold. # Do we need to hide this version? if version_index == 0 and source_path is None and version.action == 'upload': yield B2HideAction( dest_path.relative_path, dest_folder.make_full_path(dest_path.relative_path) ) # Can we start deleting? Once we start deleting, all older # versions will also be deleted. if version.action == 'hide': if keep_days < age_days: deleting = True # Delete this version if deleting: yield B2DeleteAction( dest_path.relative_path, dest_folder.make_full_path(dest_path.relative_path), version.id_, make_b2_delete_note(version, version_index, transferred), ) # Can we start deleting with the next version, based on the # age of this one? if keep_days < age_days: deleting = True b2-sdk-python-1.17.3/b2sdk/sync/policy_manager.py000066400000000000000000000074001426424117700215410ustar00rootroot00000000000000###################################################################### # # File: b2sdk/sync/policy_manager.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from ..scan.path import AbstractPath from .policy import CopyAndDeletePolicy, CopyAndKeepDaysPolicy, CopyPolicy, \ DownAndDeletePolicy, DownAndKeepDaysPolicy, DownPolicy, UpAndDeletePolicy, \ UpAndKeepDaysPolicy, UpPolicy class SyncPolicyManager: """ Policy manager; implement a logic to get a correct policy class and create a policy object based on various parameters. """ def __init__(self): self.policies = {} # dict<,> def get_policy( self, sync_type, source_path: AbstractPath, source_folder, dest_path: AbstractPath, dest_folder, now_millis, delete, keep_days, newer_file_mode, compare_threshold, compare_version_mode, encryption_settings_provider, ): """ Return a policy object. :param str sync_type: synchronization type :param b2sdk.v2.AbstractPath source_path: source file :param str source_folder: a source folder path :param b2sdk.v2.AbstractPath dest_path: destination file :param str dest_folder: a destination folder path :param int now_millis: current time in milliseconds :param bool delete: delete policy :param int keep_days: keep for days policy :param b2sdk.v2.NewerFileSyncMode newer_file_mode: setting which determines handling for destination files newer than on the source :param int compare_threshold: difference between file modification time or file size :param b2sdk.v2.CompareVersionMode compare_version_mode: setting which determines how to compare source and destination files :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: an object which decides which encryption to use (if any) :return: a policy object """ policy_class = self.get_policy_class(sync_type, delete, keep_days) return policy_class( source_path, source_folder, dest_path, dest_folder, now_millis, keep_days, newer_file_mode, compare_threshold, compare_version_mode, encryption_settings_provider, ) def get_policy_class(self, sync_type, delete, keep_days): """ Get policy class by a given sync type. :param str sync_type: synchronization type :param bool delete: if True, delete files and update from source :param int keep_days: keep for `keep_days` before delete :return: a policy class """ if sync_type == 'local-to-b2': if delete: return UpAndDeletePolicy elif keep_days: return UpAndKeepDaysPolicy else: return UpPolicy elif sync_type == 'b2-to-local': if delete: return DownAndDeletePolicy elif keep_days: return DownAndKeepDaysPolicy else: return DownPolicy elif sync_type == 'b2-to-b2': if delete: return CopyAndDeletePolicy elif keep_days: return CopyAndKeepDaysPolicy else: return CopyPolicy raise NotImplementedError( 'invalid sync type: %s, keep_days: %s, delete: %s' % ( sync_type, keep_days, delete, ) ) POLICY_MANAGER = SyncPolicyManager() b2-sdk-python-1.17.3/b2sdk/sync/report.py000066400000000000000000000137721426424117700200740ustar00rootroot00000000000000###################################################################### # # File: b2sdk/sync/report.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging import time from dataclasses import dataclass from ..progress import AbstractProgressListener from ..scan.report import ProgressReport from ..utils import format_and_scale_fraction, format_and_scale_number logger = logging.getLogger(__name__) @dataclass class SyncReport(ProgressReport): """ Handle reporting progress for syncing. Print out each file as it is processed, and puts up a sequence of progress bars. The progress bars are: - Step 1/1: count local files - Step 2/2: compare file lists - Step 3/3: transfer files This class is THREAD SAFE, so it can be used from parallel sync threads. """ def __post_init__(self): self.compare_done = False self.compare_count = 0 self.total_transfer_files = 0 # set in end_compare() self.total_transfer_bytes = 0 # set in end_compare() self.transfer_files = 0 self.transfer_bytes = 0 super().__post_init__() def _update_progress(self): if self.closed or self.no_progress: return now = time.time() interval = now - self._last_update_time if interval < self.UPDATE_INTERVAL: return self._last_update_time = now time_delta = now - self.start_time rate = 0 if time_delta == 0 else int(self.transfer_bytes / time_delta) if not self.total_done: message = ' count: %d files compare: %d files updated: %d files %s %s' % ( self.total_count, self.compare_count, self.transfer_files, format_and_scale_number(self.transfer_bytes, 'B'), format_and_scale_number(rate, 'B/s') ) # yapf: disable elif not self.compare_done: message = ' compare: %d/%d files updated: %d files %s %s' % ( self.compare_count, self.total_count, self.transfer_files, format_and_scale_number(self.transfer_bytes, 'B'), format_and_scale_number(rate, 'B/s') ) # yapf: disable else: message = ' compare: %d/%d files updated: %d/%d files %s %s' % ( self.compare_count, self.total_count, self.transfer_files, self.total_transfer_files, format_and_scale_fraction(self.transfer_bytes, self.total_transfer_bytes, 'B'), format_and_scale_number(rate, 'B/s') ) # yapf: disable self._print_line(message, False) def update_compare(self, delta): """ Report that more files have been compared. :param delta: number of files compared :type delta: int """ with self.lock: self.compare_count += delta self._update_progress() def end_compare(self, total_transfer_files, total_transfer_bytes): """ Report that the comparison has been finished. :param total_transfer_files: total number of transferred files :type total_transfer_files: int :param total_transfer_bytes: total number of transferred bytes :type total_transfer_bytes: int """ with self.lock: self.compare_done = True self.total_transfer_files = total_transfer_files self.total_transfer_bytes = total_transfer_bytes self._update_progress() def update_transfer(self, file_delta, byte_delta): """ Update transfer info. :param file_delta: number of files transferred :type file_delta: int :param byte_delta: number of bytes transferred :type byte_delta: int """ with self.lock: self.transfer_files += file_delta self.transfer_bytes += byte_delta self._update_progress() class SyncFileReporter(AbstractProgressListener): """ Listen to the progress for a single file and pass info on to a SyncReporter. """ def __init__(self, reporter, *args, **kwargs): """ :param reporter: a reporter object """ super().__init__(*args, **kwargs) self.bytes_so_far = 0 self.reporter = reporter def close(self): """ Perform a clean-up. """ # no more bytes are done, but the file is done self.reporter.update_transfer(1, 0) def set_total_bytes(self, total_byte_count): """ Set total bytes count. :param total_byte_count: total byte count :type total_byte_count: int """ pass def bytes_completed(self, byte_count): """ Set bytes completed count. :param byte_count: total byte count :type byte_count: int """ self.reporter.update_transfer(0, byte_count - self.bytes_so_far) self.bytes_so_far = byte_count def sample_sync_report_run(): """ Generate a sample report. """ import sys sync_report = SyncReport(sys.stdout, False) for i in range(20): sync_report.update_total(1) time.sleep(0.2) if i == 10: sync_report.print_completion('transferred: a.txt') if i % 2 == 0: sync_report.update_compare(1) sync_report.end_total() for i in range(10): sync_report.update_compare(1) time.sleep(0.2) if i == 3: sync_report.print_completion('transferred: b.txt') if i == 4: sync_report.update_transfer(25, 25000) sync_report.end_compare(50, 50000) for i in range(25): if i % 2 == 0: sync_report.print_completion('transferred: %d.txt' % i) sync_report.update_transfer(1, 1000) time.sleep(0.2) sync_report.close() b2-sdk-python-1.17.3/b2sdk/sync/sync.py000066400000000000000000000335101426424117700175250ustar00rootroot00000000000000###################################################################### # # File: b2sdk/sync/sync.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import concurrent.futures as futures import logging from enum import Enum, unique from ..bounded_queue_executor import BoundedQueueExecutor from ..scan.exception import InvalidArgument from ..scan.folder import AbstractFolder from ..scan.path import AbstractPath from ..scan.policies import DEFAULT_SCAN_MANAGER from ..scan.scan import zip_folders from .encryption_provider import SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, AbstractSyncEncryptionSettingsProvider from .exception import IncompleteSync from .policy import CompareVersionMode, NewerFileSyncMode from .policy_manager import POLICY_MANAGER, SyncPolicyManager from .report import SyncReport logger = logging.getLogger(__name__) def count_files(local_folder, reporter, policies_manager): """ Count all of the files in a local folder. :param b2sdk.scan.folder.AbstractFolder local_folder: a folder object. :param reporter: reporter object """ # Don't pass in a reporter to all_files. Broken symlinks will be reported # during the next pass when the source and dest files are compared. for _ in local_folder.all_files(None, policies_manager=policies_manager): reporter.update_total(1) reporter.end_total() @unique class KeepOrDeleteMode(Enum): """ Mode of dealing with old versions of files on the destination """ DELETE = 301 #: delete the old version as soon as the new one has been uploaded KEEP_BEFORE_DELETE = 302 #: keep the old versions of the file for a configurable number of days before deleting them, always keeping the newest version NO_DELETE = 303 #: keep old versions of the file, do not delete anything class Synchronizer: """ Copies multiple "files" from source to destination. Optionally deletes or hides destination files that the source does not have. The synchronizer can copy files: - From a B2 bucket to a local destination. - From a local source to a B2 bucket. - From one B2 bucket to another. - Between different folders in the same B2 bucket. It will sync only the latest versions of files. By default, the synchronizer: - Fails when the specified source directory doesn't exist or is empty. (see ``allow_empty_source`` argument) - Fails when the source is newer. (see ``newer_file_mode`` argument) - Doesn't delete a file if it's present on the destination but not on the source. (see ``keep_days_or_delete`` and ``keep_days`` arguments) - Compares files based on modification time. (see ``compare_version_mode`` and ``compare_threshold`` arguments) """ def __init__( self, max_workers, policies_manager=DEFAULT_SCAN_MANAGER, dry_run=False, allow_empty_source=False, newer_file_mode=NewerFileSyncMode.RAISE_ERROR, keep_days_or_delete=KeepOrDeleteMode.NO_DELETE, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=None, keep_days=None, sync_policy_manager: SyncPolicyManager = POLICY_MANAGER, ): """ Initialize synchronizer class and validate arguments :param int max_workers: max number of workers :param policies_manager: object which decides which files to process :param bool dry_run: test mode, does not actually transfer/delete when enabled :param bool allow_empty_source: if True, do not check whether source folder is empty :param b2sdk.v2.NewerFileSyncMode newer_file_mode: setting which determines handling for destination files newer than on the source :param b2sdk.v2.KeepOrDeleteMode keep_days_or_delete: setting which determines if we should delete or not delete or keep for `keep_days` :param b2sdk.v2.CompareVersionMode compare_version_mode: how to compare the source and destination files to find new ones :param int compare_threshold: should be greater than 0, default is 0 :param int keep_days: if keep_days_or_delete is `b2sdk.v2.KeepOrDeleteMode.KEEP_BEFORE_DELETE`, then this should be greater than 0 :param SyncPolicyManager sync_policy_manager: object which decides what to do with each file (upload, download, delete, copy, hide etc) """ self.newer_file_mode = newer_file_mode self.keep_days_or_delete = keep_days_or_delete self.keep_days = keep_days self.compare_version_mode = compare_version_mode self.compare_threshold = compare_threshold or 0 self.dry_run = dry_run self.allow_empty_source = allow_empty_source self.policies_manager = policies_manager # actually it should be called scan_policies_manager self.sync_policy_manager = sync_policy_manager self.max_workers = max_workers self._validate() def _validate(self): if self.compare_threshold < 0: raise InvalidArgument('compare_threshold', 'must be a positive integer') if self.newer_file_mode not in tuple(NewerFileSyncMode): raise InvalidArgument( 'newer_file_mode', 'must be one of :%s' % NewerFileSyncMode.__members__, ) if self.keep_days_or_delete not in tuple(KeepOrDeleteMode): raise InvalidArgument( 'keep_days_or_delete', 'must be one of :%s' % KeepOrDeleteMode.__members__, ) if self.keep_days_or_delete == KeepOrDeleteMode.KEEP_BEFORE_DELETE and self.keep_days is None: raise InvalidArgument( 'keep_days', 'is required when keep_days_or_delete is %s' % KeepOrDeleteMode.KEEP_BEFORE_DELETE, ) if self.compare_version_mode not in tuple(CompareVersionMode): raise InvalidArgument( 'compare_version_mode', 'must be one of :%s' % CompareVersionMode.__members__, ) def sync_folders( self, source_folder, dest_folder, now_millis, reporter, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ Syncs two folders. Always ensures that every file in the source is also in the destination. Deletes any file versions in the destination older than history_days. :param b2sdk.scan.folder.AbstractFolder source_folder: source folder object :param b2sdk.scan.folder.AbstractFolder dest_folder: destination folder object :param int now_millis: current time in milliseconds :param b2sdk.sync.report.SyncReport,None reporter: progress reporter :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ source_type = source_folder.folder_type() dest_type = dest_folder.folder_type() if source_type != 'b2' and dest_type != 'b2': raise ValueError('Sync between two local folders is not supported!') # For downloads, make sure that the target directory is there. if dest_type == 'local' and not self.dry_run: dest_folder.ensure_present() if source_type == 'local' and not self.allow_empty_source: source_folder.ensure_non_empty() # Make an executor to count files and run all of the actions. This is # not the same as the executor in the API object which is used for # uploads. The tasks in this executor wait for uploads. Putting them # in the same thread pool could lead to deadlock. # # We use an executor with a bounded queue to avoid using up lots of memory # when syncing lots of files. unbounded_executor = futures.ThreadPoolExecutor(max_workers=self.max_workers) queue_limit = self.max_workers + 1000 sync_executor = BoundedQueueExecutor(unbounded_executor, queue_limit=queue_limit) if source_type == 'local' and reporter is not None: # Start the thread that counts the local files. That's the operation # that should be fastest, and it provides scale for the progress reporting. sync_executor.submit(count_files, source_folder, reporter, self.policies_manager) # Bucket for scheduling actions. # For bucket-to-bucket sync, the bucket for the API calls should be the destination. action_bucket = None if dest_type == 'b2': action_bucket = dest_folder.bucket elif source_type == 'b2': action_bucket = source_folder.bucket # Schedule each of the actions. for action in self._make_folder_sync_actions( source_folder, dest_folder, now_millis, reporter, self.policies_manager, encryption_settings_provider, ): logging.debug('scheduling action %s on bucket %s', action, action_bucket) sync_executor.submit(action.run, action_bucket, reporter, self.dry_run) # Wait for everything to finish sync_executor.shutdown() if sync_executor.get_num_exceptions() != 0: raise IncompleteSync('sync is incomplete') def _make_folder_sync_actions( self, source_folder: AbstractFolder, dest_folder: AbstractFolder, now_millis: int, reporter: SyncReport, policies_manager: SyncPolicyManager = DEFAULT_SCAN_MANAGER, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ Yield a sequence of actions that will sync the destination folder to the source folder. :param b2sdk.v2.AbstractFolder source_folder: source folder object :param b2sdk.v2.AbstractFolder dest_folder: destination folder object :param int now_millis: current time in milliseconds :param b2sdk.v2.SyncReport reporter: reporter object :param b2sdk.v2.ScanPolicyManager policies_manager: object which decides which files to process :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ if self.keep_days_or_delete == KeepOrDeleteMode.KEEP_BEFORE_DELETE and dest_folder.folder_type( ) == 'local': raise InvalidArgument('keep_days_or_delete', 'cannot be used for local files') source_type = source_folder.folder_type() dest_type = dest_folder.folder_type() sync_type = '%s-to-%s' % (source_type, dest_type) if source_type != 'b2' and dest_type != 'b2': raise ValueError('Sync between two local folders is not supported!') total_files = 0 total_bytes = 0 for source_path, dest_path in zip_folders( source_folder, dest_folder, reporter, policies_manager, ): if source_path is None: logger.debug('determined that %s is not present on source', dest_path) elif dest_path is None: logger.debug('determined that %s is not present on destination', source_path) if source_path is not None: if source_type == 'b2': # For buckets we don't want to count files separately as it would require # more API calls. Instead, we count them when comparing. reporter.update_total(1) reporter.update_compare(1) for action in self._make_file_sync_actions( sync_type, source_path, dest_path, source_folder, dest_folder, now_millis, encryption_settings_provider, ): total_files += 1 total_bytes += action.get_bytes() yield action if reporter is not None: if source_type == 'b2': # For buckets we don't want to count files separately as it would require # more API calls. Instead, we count them when comparing. reporter.end_total() reporter.end_compare(total_files, total_bytes) def _make_file_sync_actions( self, sync_type: str, source_path: AbstractPath, dest_path: AbstractPath, source_folder: AbstractFolder, dest_folder: AbstractFolder, now_millis: int, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ Yields the sequence of actions needed to sync the two files :param str sync_type: synchronization type :param b2sdk.v2.AbstractPath source_path: source file object :param b2sdk.v2.AbstractPath dest_path: destination file object :param b2sdk.v2.AbstractFolder source_folder: a source folder object :param b2sdk.v2.AbstractFolder dest_folder: a destination folder object :param int now_millis: current time in milliseconds :param b2sdk.v2.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ delete = self.keep_days_or_delete == KeepOrDeleteMode.DELETE policy = self.sync_policy_manager.get_policy( sync_type, source_path, source_folder, dest_path, dest_folder, now_millis, delete, self.keep_days, self.newer_file_mode, self.compare_threshold, self.compare_version_mode, encryption_settings_provider=encryption_settings_provider, ) return policy.get_all_actions() b2-sdk-python-1.17.3/b2sdk/transfer/000077500000000000000000000000001426424117700170455ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/transfer/__init__.py000066400000000000000000000010771426424117700211630ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from .inbound.download_manager import DownloadManager from .outbound.copy_manager import CopyManager from .outbound.upload_manager import UploadManager from .emerge.emerger import Emerger __all__ = [ 'DownloadManager', 'CopyManager', 'UploadManager', 'Emerger', ] b2-sdk-python-1.17.3/b2sdk/transfer/emerge/000077500000000000000000000000001426424117700203115ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/transfer/emerge/__init__.py000066400000000000000000000004541426424117700224250ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/emerge/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/transfer/emerge/emerger.py000066400000000000000000000152251426424117700223160ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/emerge/emerger.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging from typing import Optional from b2sdk.encryption.setting import EncryptionSetting from b2sdk.file_lock import FileRetentionSetting, LegalHold from b2sdk.utils import B2TraceMetaAbstract from b2sdk.transfer.emerge.executor import EmergeExecutor from b2sdk.transfer.emerge.planner.planner import EmergePlanner logger = logging.getLogger(__name__) class Emerger(metaclass=B2TraceMetaAbstract): """ Handle complex actions around multi source copy/uploads. This class can be used to build advanced copy workflows like incremental upload. It creates a emerge plan and pass it to emerge executor - all complex logic is actually implemented in :class:`b2sdk.transfer.emerge.planner.planner.EmergePlanner` and :class:`b2sdk.transfer.emerge.executor.EmergeExecutor` """ DEFAULT_STREAMING_MAX_QUEUE_SIZE = 100 def __init__(self, services): """ :param b2sdk.v2.Services services: """ self.services = services self.emerge_executor = EmergeExecutor(services) def emerge( self, bucket_id, write_intents, file_name, content_type, file_info, progress_listener, recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, min_part_size=None, max_part_size=None, ): """ Create a new file (object in the cloud, really) from an iterable (list, tuple etc) of write intents. :param str bucket_id: a bucket ID :param write_intents: write intents to process to create a file :type write_intents: List[b2sdk.v2.WriteIntent] :param str file_name: the file name of the new B2 file :param str,None content_type: the MIME type or ``None`` to determine automatically :param dict,None file_info: a file info to store with the file or ``None`` to not store anything :param b2sdk.v2.AbstractProgressListener progress_listener: a progress listener object to use :param int min_part_size: lower limit of part size for the transfer planner, in bytes :param int max_part_size: upper limit of part size for the transfer planner, in bytes """ # WARNING: time spent trying to extract common parts of emerge() and emerge_stream() # into a separate method: 20min. You can try it too, but please increment the timer honestly. # Problematic lines are marked with a "<--". planner = self.get_emerge_planner( min_part_size=min_part_size, recommended_upload_part_size=recommended_upload_part_size, max_part_size=max_part_size, ) emerge_plan = planner.get_emerge_plan(write_intents) # <-- return self.emerge_executor.execute_emerge_plan( emerge_plan, bucket_id, file_name, content_type, file_info, progress_listener, continue_large_file_id=continue_large_file_id, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) def emerge_stream( self, bucket_id, write_intent_iterator, file_name, content_type, file_info, progress_listener, recommended_upload_part_size=None, continue_large_file_id=None, max_queue_size=DEFAULT_STREAMING_MAX_QUEUE_SIZE, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, min_part_size=None, max_part_size=None, ): """ Create a new file (object in the cloud, really) from a stream of write intents. :param str bucket_id: a bucket ID :param write_intent_iterator: iterator of :class:`~b2sdk.v2.WriteIntent` :param str file_name: the file name of the new B2 file :param str,None content_type: the MIME type or ``None`` to determine automatically :param dict,None file_info: a file info to store with the file or ``None`` to not store anything :param b2sdk.v2.AbstractProgressListener progress_listener: a progress listener object to use :param int,None recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, if ``None`` in multipart case it would always start a new large file :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param int min_part_size: lower limit of part size for the transfer planner, in bytes :param int max_part_size: upper limit of part size for the transfer planner, in bytes """ planner = self.get_emerge_planner( min_part_size=min_part_size, recommended_upload_part_size=recommended_upload_part_size, max_part_size=max_part_size, ) emerge_plan = planner.get_streaming_emerge_plan(write_intent_iterator) # <-- return self.emerge_executor.execute_emerge_plan( emerge_plan, bucket_id, file_name, content_type, file_info, progress_listener, continue_large_file_id=continue_large_file_id, max_queue_size=max_queue_size, # <-- encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) def get_emerge_planner( self, recommended_upload_part_size=None, min_part_size=None, max_part_size=None, ): return EmergePlanner.from_account_info( self.services.session.account_info, min_part_size=min_part_size, recommended_upload_part_size=recommended_upload_part_size, max_part_size=max_part_size, ) b2-sdk-python-1.17.3/b2sdk/transfer/emerge/executor.py000066400000000000000000000537011426424117700225270ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/emerge/executor.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import threading from abc import ABCMeta, abstractmethod from typing import Optional from b2sdk.encryption.setting import EncryptionSetting from b2sdk.exception import MaxFileSizeExceeded from b2sdk.file_lock import FileRetentionSetting, LegalHold, NO_RETENTION_FILE_SETTING from b2sdk.transfer.outbound.large_file_upload_state import LargeFileUploadState from b2sdk.transfer.outbound.upload_source import UploadSourceStream AUTO_CONTENT_TYPE = 'b2/x-auto' class EmergeExecutor: def __init__(self, services): self.services = services def execute_emerge_plan( self, emerge_plan, bucket_id, file_name, content_type, file_info, progress_listener, continue_large_file_id=None, max_queue_size=None, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): if emerge_plan.is_large_file(): execution = LargeFileEmergeExecution( self.services, bucket_id, file_name, content_type, file_info, progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, continue_large_file_id=continue_large_file_id, max_queue_size=max_queue_size, ) else: if continue_large_file_id is not None: raise ValueError('Cannot resume emerging single part plan.') execution = SmallFileEmergeExecution( self.services, bucket_id, file_name, content_type, file_info, progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) return execution.execute_plan(emerge_plan) class BaseEmergeExecution(metaclass=ABCMeta): DEFAULT_CONTENT_TYPE = AUTO_CONTENT_TYPE def __init__( self, services, bucket_id, file_name, content_type, file_info, progress_listener, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): self.services = services self.bucket_id = bucket_id self.file_name = file_name self.content_type = content_type self.file_info = file_info self.progress_listener = progress_listener self.encryption = encryption self.file_retention = file_retention self.legal_hold = legal_hold @abstractmethod def execute_plan(self, emerge_plan): pass class SmallFileEmergeExecution(BaseEmergeExecution): def execute_plan(self, emerge_plan): emerge_parts = list(emerge_plan.emerge_parts) assert len(emerge_parts) == 1 emerge_part = emerge_parts[0] execution_step_factory = SmallFileEmergeExecutionStepFactory(self, emerge_part) execution_step = execution_step_factory.get_execution_step() future = execution_step.execute() return future.result() class LargeFileEmergeExecution(BaseEmergeExecution): MAX_LARGE_FILE_SIZE = 10 * 1000 * 1000 * 1000 * 1000 # 10 TB def __init__( self, services, bucket_id, file_name, content_type, file_info, progress_listener, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, continue_large_file_id=None, max_queue_size=None, ): super(LargeFileEmergeExecution, self).__init__( services, bucket_id, file_name, content_type, file_info, progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) self.continue_large_file_id = continue_large_file_id self.max_queue_size = max_queue_size self._semaphore = None if self.max_queue_size is not None: self._semaphore = threading.Semaphore(self.max_queue_size) def execute_plan(self, emerge_plan): total_length = emerge_plan.get_total_length() encryption = self.encryption if total_length is not None and total_length > self.MAX_LARGE_FILE_SIZE: raise MaxFileSizeExceeded(total_length, self.MAX_LARGE_FILE_SIZE) plan_id = emerge_plan.get_plan_id() file_info = dict(self.file_info) if self.file_info is not None else {} if plan_id is not None: file_info['plan_id'] = plan_id self.progress_listener.set_total_bytes(total_length or 0) emerge_parts_dict = None if total_length is not None: emerge_parts_dict = dict(emerge_plan.enumerate_emerge_parts()) unfinished_file, finished_parts = self._get_unfinished_file_and_parts( self.bucket_id, self.file_name, file_info, self.continue_large_file_id, encryption=encryption, file_retention=self.file_retention, legal_hold=self.legal_hold, emerge_parts_dict=emerge_parts_dict, ) if unfinished_file is None: if self.content_type is None: content_type = self.DEFAULT_CONTENT_TYPE else: content_type = self.content_type unfinished_file = self.services.large_file.start_large_file( self.bucket_id, self.file_name, content_type, file_info, encryption=encryption, file_retention=self.file_retention, legal_hold=self.legal_hold, ) file_id = unfinished_file.file_id with self.progress_listener: large_file_upload_state = LargeFileUploadState(self.progress_listener) part_futures = [] for part_number, emerge_part in emerge_plan.enumerate_emerge_parts(): execution_step_factory = LargeFileEmergeExecutionStepFactory( self, emerge_part, part_number, file_id, large_file_upload_state, finished_parts=finished_parts, # it already knows encryption from BaseMergeExecution being passed as self ) execution_step = execution_step_factory.get_execution_step() future = self._execute_step(execution_step) part_futures.append(future) # Collect the sha1 checksums of the parts as the uploads finish. # If any of them raised an exception, that same exception will # be raised here by result() part_sha1_array = [f.result()['contentSha1'] for f in part_futures] # Finish the large file response = self.services.session.finish_large_file(file_id, part_sha1_array) return self.services.api.file_version_factory.from_api_response(response) def _execute_step(self, execution_step): semaphore = self._semaphore if semaphore is None: return execution_step.execute() else: semaphore.acquire() try: future = execution_step.execute() except: semaphore.release() raise else: future.add_done_callback(lambda f: semaphore.release()) return future def _get_unfinished_file_and_parts( self, bucket_id, file_name, file_info, continue_large_file_id, encryption: EncryptionSetting, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, emerge_parts_dict=None, ): if 'listFiles' not in self.services.session.account_info.get_allowed()['capabilities']: return None, {} unfinished_file = None finished_parts = {} if continue_large_file_id is not None: unfinished_file = self.services.large_file.get_unfinished_large_file( bucket_id, continue_large_file_id, prefix=file_name, ) if unfinished_file.file_info != file_info: raise ValueError( 'Cannot manually resume unfinished large file with different file_info' ) finished_parts = { part.part_number: part for part in self.services.large_file.list_parts(continue_large_file_id) } elif 'plan_id' in file_info: assert emerge_parts_dict is not None unfinished_file, finished_parts = self._find_unfinished_file_by_plan_id( bucket_id, file_name, file_info, emerge_parts_dict, encryption, file_retention, legal_hold, ) elif emerge_parts_dict is not None: unfinished_file, finished_parts = self._match_unfinished_file_if_possible( bucket_id, file_name, file_info, emerge_parts_dict, encryption, file_retention, legal_hold, ) return unfinished_file, finished_parts def _find_unfinished_file_by_plan_id( self, bucket_id, file_name, file_info, emerge_parts_dict, encryption: EncryptionSetting, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): file_retention = file_retention or NO_RETENTION_FILE_SETTING assert 'plan_id' in file_info best_match_file = None best_match_parts = {} best_match_parts_len = 0 for file_ in self.services.large_file.list_unfinished_large_files( bucket_id, prefix=file_name ): if file_.file_info != file_info: continue # FIXME: encryption is None ??? if encryption is None or file_.encryption != encryption: continue if legal_hold is None: if LegalHold.UNSET != file_.legal_hold: # Uploading and not providing legal_hold means that server's response about that file version # will have legal_hold=LegalHold.UNSET continue elif legal_hold != file_.legal_hold: continue if file_retention != file_.file_retention: # if `file_.file_retention` is UNKNOWN then we skip - lib user can still # pass UNKNOWN file_retention here - but raw_api/server won't allow it # and we don't check it here continue finished_parts = {} for part in self.services.large_file.list_parts(file_.file_id): emerge_part = emerge_parts_dict.get(part.part_number) if emerge_part is None: # large file with same `plan_id` has more parts than current plan # so we want to skip this large file because it is broken finished_parts = None break if emerge_part.is_hashable() and emerge_part.get_sha1() != part.content_sha1: continue # auto-healing - `plan_id` matches but part.sha1 doesn't - so we reupload finished_parts[part.part_number] = part if finished_parts is None: continue finished_parts_len = len(finished_parts) if best_match_file is None or finished_parts_len > best_match_parts_len: best_match_file = file_ best_match_parts = finished_parts best_match_parts_len = finished_parts_len return best_match_file, best_match_parts def _match_unfinished_file_if_possible( self, bucket_id, file_name, file_info, emerge_parts_dict, encryption: EncryptionSetting, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): """ Find an unfinished file that may be used to resume a large file upload. The file is found using the filename and comparing the uploaded parts against the local file. This is only possible if the application key being used allows ``listFiles`` access. """ file_retention = file_retention or NO_RETENTION_FILE_SETTING for file_ in self.services.large_file.list_unfinished_large_files( bucket_id, prefix=file_name ): if file_.file_name != file_name: continue if file_.file_info != file_info: continue # FIXME: what if `encryption is None` - match ANY encryption? :) if encryption is not None and encryption != file_.encryption: continue if legal_hold is None: if LegalHold.UNSET != file_.legal_hold: # Uploading and not providing legal_hold means that server's response about that file version # will have legal_hold=LegalHold.UNSET continue elif legal_hold != file_.legal_hold: continue if file_retention != file_.file_retention: # if `file_.file_retention` is UNKNOWN then we skip - lib user can still # pass UNKNOWN file_retention here - but raw_api/server won't allow it # and we don't check it here continue files_match = True finished_parts = {} for part in self.services.large_file.list_parts(file_.file_id): emerge_part = emerge_parts_dict.get(part.part_number) if emerge_part is None: files_match = False break # Compare part sizes if emerge_part.get_length() != part.content_length: files_match = False break # Compare hash assert emerge_part.is_hashable() sha1_sum = emerge_part.get_sha1() if sha1_sum != part.content_sha1: files_match = False break # Save part finished_parts[part.part_number] = part # Skip not matching files or unfinished files with no uploaded parts if not files_match or not finished_parts: continue # Return first matched file return file_, finished_parts return None, {} class BaseExecutionStepFactory(metaclass=ABCMeta): def __init__(self, emerge_execution, emerge_part): self.emerge_execution = emerge_execution self.emerge_part = emerge_part @abstractmethod def create_copy_execution_step(self, copy_range): pass @abstractmethod def create_upload_execution_step(self, stream_opener, stream_length=None, stream_sha1=None): pass def get_execution_step(self): return self.emerge_part.get_execution_step(self) class SmallFileEmergeExecutionStepFactory(BaseExecutionStepFactory): def create_copy_execution_step(self, copy_range): return CopyFileExecutionStep(self.emerge_execution, copy_range) def create_upload_execution_step(self, stream_opener, stream_length=None, stream_sha1=None): return UploadFileExecutionStep( self.emerge_execution, stream_opener, stream_length=stream_length, stream_sha1=stream_sha1 ) class LargeFileEmergeExecutionStepFactory(BaseExecutionStepFactory): def __init__( self, emerge_execution, emerge_part, part_number, large_file_id, large_file_upload_state, finished_parts=None, ): super(LargeFileEmergeExecutionStepFactory, self).__init__(emerge_execution, emerge_part) self.part_number = part_number self.large_file_id = large_file_id self.large_file_upload_state = large_file_upload_state self.finished_parts = finished_parts or {} def create_copy_execution_step(self, copy_range): return CopyPartExecutionStep( self.emerge_execution, copy_range, self.part_number, self.large_file_id, self.large_file_upload_state, ) def create_upload_execution_step(self, stream_opener, stream_length=None, stream_sha1=None): return UploadPartExecutionStep( self.emerge_execution, stream_opener, self.part_number, self.large_file_id, self.large_file_upload_state, stream_length=stream_length, stream_sha1=stream_sha1, finished_parts=self.finished_parts, ) class BaseExecutionStep(metaclass=ABCMeta): def __init__(self, emerge_execution: BaseEmergeExecution): self.emerge_execution = emerge_execution @abstractmethod def execute(self): pass class CopyFileExecutionStep(BaseExecutionStep): def __init__(self, emerge_execution, copy_source_range): super().__init__(emerge_execution) self.copy_source_range = copy_source_range def execute(self): execution = self.emerge_execution # if content type is not None then we support empty dict as default file info # but if content type is None, then setting empty dict as file info # would result with an error, because default in such case is: copy from source if execution.content_type is not None: file_info = execution.file_info or {} else: file_info = None return execution.services.copy_manager.copy_file( self.copy_source_range, execution.file_name, content_type=execution.content_type, file_info=file_info, destination_bucket_id=execution.bucket_id, progress_listener=execution.progress_listener, destination_encryption=execution.encryption, source_encryption=self.copy_source_range.encryption, file_retention=execution.file_retention, legal_hold=execution.legal_hold, ) class CopyPartExecutionStep(BaseExecutionStep): def __init__( self, emerge_execution, copy_source_range, part_number, large_file_id, large_file_upload_state, finished_parts=None ): super().__init__(emerge_execution) self.copy_source_range = copy_source_range self.part_number = part_number self.large_file_id = large_file_id self.large_file_upload_state = large_file_upload_state self.finished_parts = finished_parts or {} def execute(self): return self.emerge_execution.services.copy_manager.copy_part( self.large_file_id, self.copy_source_range, self.part_number, self.large_file_upload_state, finished_parts=self.finished_parts, destination_encryption=self.emerge_execution.encryption, source_encryption=self.copy_source_range.encryption, ) class UploadFileExecutionStep(BaseExecutionStep): def __init__(self, emerge_execution, stream_opener, stream_length=None, stream_sha1=None): super().__init__(emerge_execution) self.stream_opener = stream_opener self.stream_length = stream_length self.stream_sha1 = stream_sha1 def execute(self): upload_source = UploadSourceStream( self.stream_opener, stream_length=self.stream_length, stream_sha1=self.stream_sha1, ) execution = self.emerge_execution return execution.services.upload_manager.upload_file( execution.bucket_id, upload_source, execution.file_name, execution.content_type or execution.DEFAULT_CONTENT_TYPE, execution.file_info or {}, execution.progress_listener, encryption=execution.encryption, file_retention=execution.file_retention, legal_hold=execution.legal_hold, ) class UploadPartExecutionStep(BaseExecutionStep): def __init__( self, emerge_execution, stream_opener, part_number, large_file_id, large_file_upload_state, stream_length=None, stream_sha1=None, finished_parts=None ): super().__init__(emerge_execution) self.stream_opener = stream_opener self.stream_length = stream_length self.stream_sha1 = stream_sha1 self.part_number = part_number self.large_file_id = large_file_id self.large_file_upload_state = large_file_upload_state self.finished_parts = finished_parts or {} def execute(self): execution = self.emerge_execution upload_source = UploadSourceStream( self.stream_opener, stream_length=self.stream_length, stream_sha1=self.stream_sha1, ) return execution.services.upload_manager.upload_part( execution.bucket_id, self.large_file_id, upload_source, self.part_number, self.large_file_upload_state, finished_parts=self.finished_parts, encryption=execution.encryption, ) b2-sdk-python-1.17.3/b2sdk/transfer/emerge/planner/000077500000000000000000000000001426424117700217505ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/transfer/emerge/planner/__init__.py000066400000000000000000000004641426424117700240650ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/emerge/planner/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/transfer/emerge/planner/part_definition.py000066400000000000000000000123051426424117700255010ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/emerge/planner/part_definition.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import ABCMeta, abstractmethod from functools import partial from b2sdk.stream.chained import ChainedStream from b2sdk.stream.range import wrap_with_range from b2sdk.utils import hex_sha1_of_unlimited_stream class BaseEmergePartDefinition(metaclass=ABCMeta): @abstractmethod def get_length(self): pass @abstractmethod def get_part_id(self): pass @abstractmethod def get_execution_step(self, execution_step_factory): pass def is_hashable(self): return False def get_sha1(self): return None class UploadEmergePartDefinition(BaseEmergePartDefinition): def __init__(self, upload_source, relative_offset, length): self.upload_source = upload_source self.relative_offset = relative_offset self.length = length self._sha1 = None def __repr__(self): return ( '<{classname} upload_source={upload_source} relative_offset={relative_offset} ' 'length={length}>' ).format( classname=self.__class__.__name__, upload_source=repr(self.upload_source), relative_offset=self.relative_offset, length=self.length, ) def get_length(self): return self.length def get_part_id(self): return self.get_sha1() def is_hashable(self): return True def get_sha1(self): if self._sha1 is None: if self.relative_offset == 0 and self.length == self.upload_source.get_content_length(): # this is part is equal to whole upload source - so we use `get_content_sha1()` # and if sha1 is already given, we skip computing it again self._sha1 = self.upload_source.get_content_sha1() else: with self._get_stream() as stream: self._sha1, _ = hex_sha1_of_unlimited_stream(stream) return self._sha1 def get_execution_step(self, execution_step_factory): return execution_step_factory.create_upload_execution_step( self._get_stream, stream_length=self.length, stream_sha1=self.get_sha1(), ) def _get_stream(self): fp = self.upload_source.open() return wrap_with_range( fp, self.upload_source.get_content_length(), self.relative_offset, self.length ) class UploadSubpartsEmergePartDefinition(BaseEmergePartDefinition): def __init__(self, upload_subparts): self.upload_subparts = upload_subparts self._is_hashable = all(subpart.is_hashable() for subpart in upload_subparts) self._sha1 = None def __repr__(self): return '<{classname} upload_subparts={upload_subparts}>'.format( classname=self.__class__.__name__, upload_subparts=repr(self.upload_subparts), ) def get_length(self): return sum(subpart.length for subpart in self.upload_subparts) def get_part_id(self): if self.is_hashable(): return self.get_sha1() else: return tuple(subpart.get_subpart_id() for subpart in self.upload_subparts) def is_hashable(self): return self._is_hashable def get_sha1(self): if self._sha1 is None and self.is_hashable(): with self._get_stream() as stream: self._sha1, _ = hex_sha1_of_unlimited_stream(stream) return self._sha1 def get_execution_step(self, execution_step_factory): return execution_step_factory.create_upload_execution_step( partial(self._get_stream, emerge_execution=execution_step_factory.emerge_execution), stream_length=self.get_length(), stream_sha1=self.get_sha1(), ) def _get_stream(self, emerge_execution=None): return ChainedStream( [ subpart.get_stream_opener(emerge_execution=emerge_execution) for subpart in self.upload_subparts ] ) class CopyEmergePartDefinition(BaseEmergePartDefinition): def __init__(self, copy_source, relative_offset, length): self.copy_source = copy_source self.relative_offset = relative_offset self.length = length def __repr__(self): return ( '<{classname} copy_source={copy_source} relative_offset={relative_offset} ' 'length={length}>' ).format( classname=self.__class__.__name__, copy_source=repr(self.copy_source), relative_offset=self.relative_offset, length=self.length, ) def get_length(self): return self.length def get_part_id(self): return (self.copy_source.file_id, self.relative_offset, self.length) def get_execution_step(self, execution_step_factory): return execution_step_factory.create_copy_execution_step( self.copy_source.get_copy_source_range(self.relative_offset, self.length) ) b2-sdk-python-1.17.3/b2sdk/transfer/emerge/planner/planner.py000066400000000000000000000727611426424117700237760ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/emerge/planner/planner.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from math import ceil import hashlib import json from abc import ABCMeta, abstractmethod from collections import deque from itertools import chain from b2sdk.transfer.emerge.planner.part_definition import ( CopyEmergePartDefinition, UploadEmergePartDefinition, UploadSubpartsEmergePartDefinition, ) from b2sdk.transfer.emerge.planner.upload_subpart import ( LocalSourceUploadSubpart, RemoteSourceUploadSubpart, ) MEGABYTE = 1000 * 1000 GIGABYTE = 1000 * MEGABYTE class UploadBuffer: """ data container used by EmergePlanner for temporary storage of write intents """ def __init__(self, start_offset, buff=None): self._start_offset = start_offset self._buff = buff or [] if self._buff: self._end_offset = self._buff[-1][1] else: self._end_offset = self._start_offset @property def start_offset(self): return self._start_offset @property def end_offset(self): return self._end_offset @property def length(self): return self.end_offset - self.start_offset def intent_count(self): return len(self._buff) def get_intent(self, index): return self.get_item(index)[0] def get_item(self, index): return self._buff[index] def iter_items(self): return iter(self._buff) def append(self, intent, fragment_end): self._buff.append((intent, fragment_end)) self._end_offset = fragment_end def get_slice(self, start_idx=None, end_idx=None, start_offset=None): start_idx = start_idx or 0 buff_slice = self._buff[start_idx:end_idx] if start_offset is None: if start_idx == 0: start_offset = self.start_offset else: start_offset = self._buff[start_idx - 1:start_idx][0][1] return self.__class__(start_offset, buff_slice) class EmergePlanner: """ Creates a list of actions required for advanced creation of an object in the cloud from an iterator of write intent objects """ DEFAULT_MIN_PART_SIZE = 5 * MEGABYTE DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE = 100 * MEGABYTE DEFAULT_MAX_PART_SIZE = 5 * GIGABYTE def __init__( self, min_part_size=None, recommended_upload_part_size=None, max_part_size=None, ): self.min_part_size = min_part_size or self.DEFAULT_MIN_PART_SIZE self.recommended_upload_part_size = recommended_upload_part_size or self.DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE self.max_part_size = max_part_size or self.DEFAULT_MAX_PART_SIZE assert self.min_part_size <= self.recommended_upload_part_size <= self.max_part_size @classmethod def from_account_info( cls, account_info, min_part_size=None, recommended_upload_part_size=None, max_part_size=None ): if recommended_upload_part_size is None: recommended_upload_part_size = account_info.get_recommended_part_size() if min_part_size is None and recommended_upload_part_size < cls.DEFAULT_MIN_PART_SIZE: min_part_size = recommended_upload_part_size if max_part_size is None and recommended_upload_part_size > cls.DEFAULT_MAX_PART_SIZE: max_part_size = recommended_upload_part_size kwargs = { 'min_part_size': min_part_size, 'recommended_upload_part_size': recommended_upload_part_size, 'max_part_size': max_part_size, } return cls(**{key: value for key, value in kwargs.items() if value is not None}) def get_emerge_plan(self, write_intents): write_intents = sorted(write_intents, key=lambda intent: intent.destination_offset) # the upload part size recommended by the server causes errors with files larger than 1TB # (with the current 100MB part size and 10000 part count limit). # Therefore here we increase the recommended upload part size if needed. # the constant is for handling mixed upload/copy in concatenate etc max_destination_offset = max(intent.destination_end_offset for intent in write_intents) self.recommended_upload_part_size = max( self.recommended_upload_part_size, min( ceil(1.5 * max_destination_offset / 10000), self.max_part_size, ) ) assert self.min_part_size <= self.recommended_upload_part_size <= self.max_part_size, ( self.min_part_size, self.recommended_upload_part_size, self.max_part_size ) return self._get_emerge_plan(write_intents, EmergePlan) def get_streaming_emerge_plan(self, write_intent_iterator): return self._get_emerge_plan(write_intent_iterator, StreamingEmergePlan) def _get_emerge_plan(self, write_intent_iterator, plan_class): return plan_class( self._get_emerge_parts( self._select_intent_fragments(self._validatation_iterator(write_intent_iterator)) ) ) def _get_emerge_parts(self, intent_fragments_iterator): # This is where the magic happens. Instead of describing typical inputs and outputs here, # We've put them in tests. It is recommended to read those tests before trying to comprehend # the implementation details of this function. min_part_size = self.min_part_size # this stores current intent that we need to process - we may get # it in fragments so we want to glue just by updating `current_end` current_intent = None current_end = 0 upload_buffer = UploadBuffer(0) first = True last = False for intent, fragment_end in intent_fragments_iterator: if current_intent is None: # this is a first loop run - just initialize current intent current_intent = intent current_end = fragment_end continue if intent is current_intent: # new intent is the same as previously processed intent, so lets glue them together # this happens when the caller splits N overlapping intents into overlapping fragments # and two fragments from the same intent end up streaming into here one after the other current_end = fragment_end continue if intent is None: last = True # incoming intent is different - this means that now we have to decide what to do: # if this is a copy intent and we want to copy it server-side, then we have to # flush the whole upload buffer we accumulated so far, but OTOH we may decide that we just want to # append it to upload buffer (see complete, untrivial logic below) and then maybe # flush some upload parts from upload buffer (if there is enough in the buffer) current_len = current_end - upload_buffer.end_offset # should we flush the upload buffer or do we have to add a chunk of the copy first? if current_intent.is_copy() and current_len >= min_part_size: # check if we can flush upload buffer or there is some missing bytes to fill it to `min_part_size` if upload_buffer.intent_count() > 0 and upload_buffer.length < min_part_size: missing_length = min_part_size - upload_buffer.length else: missing_length = 0 if missing_length > 0 and current_len - missing_length < min_part_size: # current intent is *not* a "small copy", but upload buffer is small # and current intent is too short with the buffer to reach the minimum part size # so we append current intent to upload buffer upload_buffer.append(current_intent, current_end) else: if missing_length > 0: # we "borrow" a fragment of current intent to upload buffer # to fill it to minimum part size upload_buffer.append( current_intent, upload_buffer.end_offset + missing_length ) # completely flush the upload buffer for upload_buffer_part in self._buff_split(upload_buffer): yield self._get_upload_part(upload_buffer_part) # split current intent (copy source) to parts and yield copy_parts = self._get_copy_parts( current_intent, start_offset=upload_buffer.end_offset, end_offset=current_end, ) for part in copy_parts: yield part upload_buffer = UploadBuffer(current_end) else: if current_intent.is_copy() and first and last: # this is a single intent "small copy" - we force use of `copy_file` copy_parts = self._get_copy_parts( current_intent, start_offset=upload_buffer.end_offset, end_offset=current_end, ) for part in copy_parts: yield part else: # this is a upload source or "small copy" source (that is *not* single intent) # either way we just add them to upload buffer upload_buffer.append(current_intent, current_end) upload_buffer_parts = list(self._buff_split(upload_buffer)) # we flush all parts excluding last one - we may want to extend # this last part with "incoming" intent in next loop run for upload_buffer_part in upload_buffer_parts[:-1]: yield self._get_upload_part(upload_buffer_part) upload_buffer = upload_buffer_parts[-1] current_intent = intent first = False current_end = fragment_end if current_intent is None: # this is a sentinel - there would be no more fragments - we have to flush upload buffer for upload_buffer_part in self._buff_split(upload_buffer): yield self._get_upload_part(upload_buffer_part) def _get_upload_part(self, upload_buffer): """ Build emerge part from upload buffer. """ if upload_buffer.intent_count() == 1 and upload_buffer.get_intent(0).is_upload(): intent = upload_buffer.get_intent(0) relative_offset = upload_buffer.start_offset - intent.destination_offset length = upload_buffer.length definition = UploadEmergePartDefinition(intent.outbound_source, relative_offset, length) else: subparts = [] fragment_start = upload_buffer.start_offset for intent, fragment_end in upload_buffer.iter_items(): relative_offset = fragment_start - intent.destination_offset length = fragment_end - fragment_start if intent.is_upload(): subpart_class = LocalSourceUploadSubpart elif intent.is_copy(): subpart_class = RemoteSourceUploadSubpart else: raise RuntimeError('This cannot happen!!!') subparts.append(subpart_class(intent.outbound_source, relative_offset, length)) fragment_start = fragment_end definition = UploadSubpartsEmergePartDefinition(subparts) return EmergePart(definition) def _get_copy_parts(self, copy_intent, start_offset, end_offset): """ Split copy intent to emerge parts. """ fragment_length = end_offset - start_offset part_count = int(fragment_length / self.max_part_size) last_part_length = fragment_length % self.max_part_size if last_part_length == 0: last_part_length = self.max_part_size else: part_count += 1 if part_count == 1: part_sizes = [last_part_length] else: if last_part_length < int(fragment_length / (part_count + 1)): part_count += 1 base_part_size = int(fragment_length / part_count) size_remainder = fragment_length % part_count part_sizes = [ base_part_size + (1 if i < size_remainder else 0) for i in range(part_count) ] copy_source = copy_intent.outbound_source relative_offset = start_offset - copy_intent.destination_offset for part_size in part_sizes: yield EmergePart(CopyEmergePartDefinition(copy_source, relative_offset, part_size)) relative_offset += part_size def _buff_split(self, upload_buffer): """ Split upload buffer to parts candidates - smaller upload buffers. :rtype iterator[b2sdk.transfer.emerge.planner.planner.UploadBuffer]: """ if upload_buffer.intent_count() == 0: return tail_buffer = upload_buffer while True: if tail_buffer.length < self.recommended_upload_part_size + self.min_part_size: # `EmergePlanner_buff_partition` can split in such way that tail part # can be smaller than `min_part_size` - to avoid unnecessary download of possible # incoming copy intent, we don't split further yield tail_buffer return head_buff, tail_buffer = self._buff_partition(tail_buffer) yield head_buff def _buff_partition(self, upload_buffer): """ Split upload buffer to two parts (smaller upload buffers). In result left part cannot be split more, and nothing can be assumed about right part. :rtype tuple(b2sdk.transfer.emerge.planner.planner.UploadBuffer, b2sdk.transfer.emerge.planner.planner.UploadBuffer): """ left_buff = UploadBuffer(upload_buffer.start_offset) buff_start = upload_buffer.start_offset for idx, (intent, fragment_end) in enumerate(upload_buffer.iter_items()): candidate_size = fragment_end - buff_start if candidate_size > self.recommended_upload_part_size: right_fragment_size = candidate_size - self.recommended_upload_part_size left_buff.append(intent, fragment_end - right_fragment_size) return left_buff, upload_buffer.get_slice( start_idx=idx, start_offset=left_buff.end_offset ) else: left_buff.append(intent, fragment_end) if candidate_size == self.recommended_upload_part_size: return left_buff, upload_buffer.get_slice(start_idx=idx + 1) return left_buff, UploadBuffer(left_buff.end_offset) def _select_intent_fragments(self, write_intent_iterator): """ Select overapping write intent fragments to use. To solve overlapping intents selection, intents can be split to smaller fragments. Those fragments are yieled as soon as decision can be made to use them, so there is possibility that one intent is yielded in multiple fragments. Those would be merged again by higher level iterator that produces emerge parts, but in principle this merging can happen here. Not merging it is a code design decision to make this function easier to implement and also it would allow yielding emerge parts a bit quicker. """ # `protected_intent_length` for upload state is 0, so it would generate at most single intent fragment # every loop iteration, but algorithm is not assuming that - one may one day choose to # protect upload fragments length too - eg. to avoid situation when file is opened to # read just small number of bytes and then switch to another overlapping upload source upload_intents_state = IntentsState() copy_intents_state = IntentsState(protected_intent_length=self.min_part_size) last_sent_offset = 0 incoming_offset = None while True: incoming_intent = next(write_intent_iterator, None) if incoming_intent is None: incoming_offset = None else: incoming_offset = incoming_intent.destination_offset upload_intents = list( upload_intents_state.state_update(last_sent_offset, incoming_offset) ) copy_intents = list(copy_intents_state.state_update(last_sent_offset, incoming_offset)) intent_fragments = self._merge_intent_fragments( last_sent_offset, upload_intents, copy_intents, ) for intent, intent_fragment_end in intent_fragments: yield intent, intent_fragment_end last_sent_offset = intent_fragment_end if incoming_offset is not None and last_sent_offset < incoming_offset: raise ValueError( 'Cannot emerge file with holes. ' 'Found hole range: ({}, {})'.format(last_sent_offset, incoming_offset) ) if incoming_intent is None: yield None, None # lets yield sentinel for cleaner `_get_emerge_parts` implementation return if incoming_intent.is_upload(): upload_intents_state.add(incoming_intent) elif incoming_intent.is_copy(): copy_intents_state.add(incoming_intent) else: raise RuntimeError('This should not happen at all!') def _merge_intent_fragments(self, start_offset, upload_intents, copy_intents): """ Select "competing" upload and copy fragments. Upload and copy fragments may overlap so we need to choose right one to use - copy fragments are prioritized unless this fragment is unprotected (we use "protection" as an abstract for "short copy" fragments - meaning upload fragments have higher priority than "short copy") """ upload_intents = deque(upload_intents) copy_intents = deque(copy_intents) while True: upload_intent = copy_intent = None if upload_intents: upload_intent, upload_end, _ = upload_intents[0] if copy_intents: copy_intent, copy_end, copy_protected = copy_intents[0] if upload_intent is not None and copy_intent is not None: if not copy_protected: yield_intent = upload_intent else: yield_intent = copy_intent start_offset = min(upload_end, copy_end) yield yield_intent, start_offset if start_offset >= upload_end: upload_intents.popleft() if start_offset >= copy_end: copy_intents.popleft() elif upload_intent is not None: yield upload_intent, upload_end upload_intents.popleft() elif copy_intent is not None: yield copy_intent, copy_end copy_intents.popleft() else: return def _validatation_iterator(self, write_intents): """ Iterate over write intents and validate length and order. """ last_offset = 0 for write_intent in write_intents: if write_intent.length is None: raise ValueError('Planner cannot support write intents of unknown length') if write_intent.destination_offset < last_offset: raise ValueError('Write intent stream have to be sorted by destination offset') last_offset = write_intent.destination_offset yield write_intent class IntentsState: """ Store and process state of incoming write intents to solve overlapping intents selection in streaming manner. It does not check if intents are of the same kind (upload/copy), but the intention was to use it to split incoming intents by kind (two intents state are required then). If there would be no need for differentiating incoming intents, this would still work - so intent kind is ignored at this level. To address "short copy" prioritization problem (and avoidance) - ``protected_intent_length`` param was introduced to prevent logic from allowing too small fragments (if it is possible) """ def __init__(self, protected_intent_length=0): self.protected_intent_length = protected_intent_length self._current_intent = None self._next_intent = None self._last_sent_offset = 0 self._incoming_offset = None self._current_intent_start = None self._current_intent_end = None self._next_intent_end = None def add(self, incoming_intent): """ Add incoming intent to state. It has to called *after* ``IntentsState.state_update`` but it is not verified. """ if self._next_intent is None: self._set_next_intent(incoming_intent) elif incoming_intent.destination_end_offset > self._next_intent_end: # here either incoming intent starts at the same position as next intent # (and current intent is None in such case - it was just cleared in `state_update` # or it was cleared some time ago - in previous iteratios) or we are in situation # when current and next intent overlaps, and `last_sent_offset` is now set to # incoming intent `destination_offset` - in both cases we want to choose # intent which has larger `destination_end_offset` self._set_next_intent(incoming_intent) def state_update(self, last_sent_offset, incoming_offset): """ Update the state using incoming intent offset. It has to be called *before* ``IntentsState.add`` and even if incoming intent would not be added to this intents state. It would yield a state of this stream of intents (like copy or upload) from ``last_sent_offset`` to ``incoming_offset``. So here happens the first stage of solving overlapping intents selection - but write intent iterator can be split to multiple substreams (like copy and upload) so additional stage is required to cover this. """ if self._current_intent is not None: if last_sent_offset >= self._current_intent_end: self._set_current_intent(None, None) # `effective_incoming_offset` is a safeguard after intent iterator is drained if incoming_offset is not None: effective_incoming_offset = incoming_offset elif self._next_intent is not None: effective_incoming_offset = self._next_intent_end elif self._current_intent is not None: effective_incoming_offset = self._current_intent_end else: # intent iterator is drained and this state is empty return if ( self._current_intent is None and self._next_intent is not None and ( self._next_intent.destination_offset != effective_incoming_offset or incoming_offset is None ) ): self._set_current_intent(self._next_intent, last_sent_offset) self._set_next_intent(None) # current and next can be both not None at this point only if they overlap if ( self._current_intent is not None and self._next_intent is not None and effective_incoming_offset > self._current_intent_end ): # incoming intent does not overlap with current intent # so we switch to next because we are sure that we will have to use it anyway # (of course other overriding (eg. "copy" over "upload") state can have # overlapping intent but we have no information about it here) # but we also need to protect current intent length if not self._is_current_intent_protected(): # we were unable to protect current intent, so we can safely rotate self._set_current_intent(self._next_intent, last_sent_offset) self._set_next_intent(None) else: remaining_len = self.protected_intent_length - ( last_sent_offset - self._current_intent_start ) if remaining_len > 0: last_sent_offset += remaining_len if not self._can_be_protected(last_sent_offset, self._next_intent_end): last_sent_offset = self._current_intent_end yield self._current_intent, last_sent_offset, True self._set_current_intent(self._next_intent, last_sent_offset) self._set_next_intent(None) if self._current_intent is not None: yield ( self._current_intent, min(effective_incoming_offset, self._current_intent_end), self._is_current_intent_protected(), ) def _set_current_intent(self, intent, start_offset): self._current_intent = intent if self._current_intent is not None: self._current_intent_end = self._current_intent.destination_end_offset else: self._current_intent_end = None assert start_offset is None self._current_intent_start = start_offset def _set_next_intent(self, intent): self._next_intent = intent if self._next_intent is not None: self._next_intent_end = self._next_intent.destination_end_offset else: self._next_intent_end = None def _is_current_intent_protected(self): """ States if current intent is protected. Intent can be split to smaller fragments, but to choose upload over "small copy" we need to know for fragment if it is a "small copy" or not. In result of solving overlapping intents selection there might be a situation when original intent was not a small copy, but in effect it will be used only partially and in effect it may be a "small copy". Algorithm attempts to avoid using smaller fragments than ``protected_intent_length`` but sometimes it may be impossible. So if this function returns ``False`` it means that used length of this intent is smaller than ``protected_intent_length`` and the algorithm was unable to avoid this. """ return self._can_be_protected(self._current_intent_start, self._current_intent_end) def _can_be_protected(self, start, end): return end - start >= self.protected_intent_length class BaseEmergePlan(metaclass=ABCMeta): def __init__(self, emerge_parts): self.emerge_parts = emerge_parts @abstractmethod def is_large_file(self): pass @abstractmethod def get_total_length(self): pass @abstractmethod def get_plan_id(self): pass def enumerate_emerge_parts(self): return enumerate(self.emerge_parts, 1) class EmergePlan(BaseEmergePlan): def __init__(self, emerge_parts): super(EmergePlan, self).__init__(list(emerge_parts)) self._is_large_file = len(self.emerge_parts) > 1 def is_large_file(self): return self._is_large_file def get_total_length(self): return sum(emerge_part.get_length() for emerge_part in self.emerge_parts) def get_plan_id(self): if all(part.is_hashable() for part in self.emerge_parts): return None json_id = json.dumps([emerge_part.get_part_id() for emerge_part in self.emerge_parts]) return hashlib.sha1(json_id.encode()).hexdigest() class StreamingEmergePlan(BaseEmergePlan): def __init__(self, emerge_parts_iterator): emerge_parts, self._is_large_file = self._peek_for_large_file(emerge_parts_iterator) super(StreamingEmergePlan, self).__init__(emerge_parts) def is_large_file(self): return self._is_large_file def get_total_length(self): return None def get_plan_id(self): return None def _peek_for_large_file(self, emerge_parts_iterator): first_part = next(emerge_parts_iterator, None) if first_part is None: raise ValueError('Empty emerge parts iterator') second_part = next(emerge_parts_iterator, None) if second_part is None: return iter([first_part]), False else: return chain([first_part, second_part], emerge_parts_iterator), True class EmergePart: def __init__(self, part_definition, verification_ranges=None): self.part_definition = part_definition self.verification_ranges = verification_ranges def __repr__(self): return '<{classname} part_definition={part_definition}>'.format( classname=self.__class__.__name__, part_definition=repr(self.part_definition), ) def get_length(self): return self.part_definition.get_length() def get_execution_step(self, execution_step_factory): return self.part_definition.get_execution_step(execution_step_factory) def get_part_id(self): return self.part_definition.get_part_id() def is_hashable(self): return self.part_definition.is_hashable() def get_sha1(self): return self.part_definition.get_sha1() b2-sdk-python-1.17.3/b2sdk/transfer/emerge/planner/upload_subpart.py000066400000000000000000000067231426424117700253560ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/emerge/planner/upload_subpart.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import io from abc import ABCMeta, abstractmethod from functools import partial from b2sdk.stream.chained import StreamOpener from b2sdk.stream.range import wrap_with_range from b2sdk.utils import hex_sha1_of_unlimited_stream class BaseUploadSubpart(metaclass=ABCMeta): def __init__(self, outbound_source, relative_offset, length): self.outbound_source = outbound_source self.relative_offset = relative_offset self.length = length def __repr__(self): return ( '<{classname} outbound_source={outbound_source} relative_offset={relative_offset} ' 'length={length}>' ).format( classname=self.__class__.__name__, outbound_source=repr(self.outbound_source), relative_offset=self.relative_offset, length=self.length, ) @abstractmethod def get_subpart_id(self): pass @abstractmethod def get_stream_opener(self, emerge_execution=None): pass def is_hashable(self): return False class RemoteSourceUploadSubpart(BaseUploadSubpart): def __init__(self, *args, **kwargs): super(RemoteSourceUploadSubpart, self).__init__(*args, **kwargs) self._download_buffer_cache = None def get_subpart_id(self): return (self.outbound_source.file_id, self.relative_offset, self.length) def get_stream_opener(self, emerge_execution=None): if emerge_execution is None: raise RuntimeError('Cannot open remote source without emerge execution instance.') return CachedBytesStreamOpener(partial(self._download, emerge_execution)) def _download(self, emerge_execution): url = emerge_execution.services.session.get_download_url_by_id(self.outbound_source.file_id) absolute_offset = self.outbound_source.offset + self.relative_offset range_ = (absolute_offset, absolute_offset + self.length - 1) with io.BytesIO() as bytes_io: downloaded_file = emerge_execution.services.download_manager.download_file_from_url( url, range_=range_, encryption=self.outbound_source.encryption ) downloaded_file.save(bytes_io) return bytes_io.getvalue() class LocalSourceUploadSubpart(BaseUploadSubpart): def get_subpart_id(self): with self._get_stream() as stream: sha1, _ = hex_sha1_of_unlimited_stream(stream) return sha1 def get_stream_opener(self, emerge_execution=None): return self._get_stream def _get_stream(self): fp = self.outbound_source.open() return wrap_with_range( fp, self.outbound_source.get_content_length(), self.relative_offset, self.length ) def is_hashable(self): return True class CachedBytesStreamOpener(StreamOpener): def __init__(self, bytes_data_callback): self.bytes_data_callback = bytes_data_callback self._bytes_data_cache = None def __call__(self): if self._bytes_data_cache is None: self._bytes_data_cache = self.bytes_data_callback() return io.BytesIO(self._bytes_data_cache) def cleanup(self): self._bytes_data_cache = None b2-sdk-python-1.17.3/b2sdk/transfer/emerge/write_intent.py000066400000000000000000000052561426424117700234060ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/emerge/write_intent.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### class WriteIntent: """ Wrapper for outbound source that defines destination offset. """ def __init__(self, outbound_source, destination_offset=0): """ :param b2sdk.v2.OutboundTransferSource outbound_source: data source (remote or local) :param int destination_offset: point of start in destination file """ if outbound_source.get_content_length() is None: raise ValueError('Cannot wrap outbound source of unknown length') self.outbound_source = outbound_source self.destination_offset = destination_offset def __repr__(self): return ( '<{classname} outbound_source={outbound_source} ' 'destination_offset={destination_offset} id={id}>' ).format( classname=self.__class__.__name__, outbound_source=repr(self.outbound_source), destination_offset=self.destination_offset, id=id(self), ) @property def length(self): """ Length of the write intent. :rtype: int """ return self.outbound_source.get_content_length() @property def destination_end_offset(self): """ Offset of source end in destination file. :rtype: int """ return self.destination_offset + self.length def is_copy(self): """ States if outbound source is remote source and requires copying. :rtype: bool """ return self.outbound_source.is_copy() def is_upload(self): """ States if outbound source is local source and requires uploading. :rtype: bool """ return self.outbound_source.is_upload() @classmethod def wrap_sources_iterator(cls, outbound_sources_iterator): """ Helper that wraps outbound sources iterator with write intents. Can be used in cases similar to concatenate to automatically compute destination offsets :param: iterator[b2sdk.v2.OutboundTransferSource] outbound_sources_iterator: iterator of outbound sources :rtype: generator[b2sdk.v2.WriteIntent] """ current_position = 0 for outbound_source in outbound_sources_iterator: length = outbound_source.get_content_length() write_intent = WriteIntent(outbound_source, destination_offset=current_position) current_position += length yield write_intent b2-sdk-python-1.17.3/b2sdk/transfer/inbound/000077500000000000000000000000001426424117700205035ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/transfer/inbound/__init__.py000066400000000000000000000004551426424117700226200ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/inbound/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/transfer/inbound/download_manager.py000066400000000000000000000101421426424117700243540ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/inbound/download_manager.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging from typing import Optional from b2sdk.encryption.setting import EncryptionSetting from b2sdk.progress import DoNothingProgressListener from b2sdk.exception import ( InvalidRange, ) from b2sdk.utils import B2TraceMetaAbstract from .downloaded_file import DownloadedFile from .downloader.parallel import ParallelDownloader from .downloader.simple import SimpleDownloader from ..transfer_manager import TransferManager from ...utils.thread_pool import ThreadPoolMixin logger = logging.getLogger(__name__) class DownloadManager(TransferManager, ThreadPoolMixin, metaclass=B2TraceMetaAbstract): """ Handle complex actions around downloads to free raw_api from that responsibility. """ # minimum size of a download chunk DEFAULT_MIN_PART_SIZE = 100 * 1024 * 1024 # block size used when downloading file. If it is set to a high value, # progress reporting will be jumpy, if it's too low, it impacts CPU MIN_CHUNK_SIZE = 8192 # ~1MB file will show ~1% progress increment MAX_CHUNK_SIZE = 1024**2 PARALLEL_DOWNLOADER_CLASS = staticmethod(ParallelDownloader) SIMPLE_DOWNLOADER_CLASS = staticmethod(SimpleDownloader) def __init__(self, write_buffer_size: Optional[int] = None, check_hash: bool = True, **kwargs): """ Initialize the DownloadManager using the given services object. """ super().__init__(**kwargs) self.strategies = [ self.PARALLEL_DOWNLOADER_CLASS( min_part_size=self.DEFAULT_MIN_PART_SIZE, min_chunk_size=self.MIN_CHUNK_SIZE, max_chunk_size=max(self.MAX_CHUNK_SIZE, write_buffer_size or 0), align_factor=write_buffer_size, thread_pool=self._thread_pool, check_hash=check_hash, ), self.SIMPLE_DOWNLOADER_CLASS( min_chunk_size=self.MIN_CHUNK_SIZE, max_chunk_size=max(self.MAX_CHUNK_SIZE, write_buffer_size or 0), align_factor=write_buffer_size, thread_pool=self._thread_pool, check_hash=check_hash, ), ] self.write_buffer_size = write_buffer_size self.check_hash = check_hash def download_file_from_url( self, url, progress_listener=None, range_=None, encryption: Optional[EncryptionSetting] = None, ) -> DownloadedFile: """ :param url: url from which the file should be downloaded :param progress_listener: where to notify about downloading progress :param range_: 2-element tuple containing data of http Range header :param b2sdk.v2.EncryptionSetting encryption: encryption setting (``None`` if unknown) """ progress_listener = progress_listener or DoNothingProgressListener() with self.services.session.download_file_from_url( url, range_=range_, encryption=encryption, ) as response: download_version = self.services.api.download_version_factory.from_response_headers( response.headers ) if range_ is not None: # 2021-05-20: unfortunately for a read of a complete object server does not return the 'Content-Range' header if (range_[1] - range_[0] + 1) != download_version.content_length: raise InvalidRange(download_version.content_length, range_) return DownloadedFile( download_version=download_version, download_manager=self, range_=range_, response=response, encryption=encryption, progress_listener=progress_listener, write_buffer_size=self.write_buffer_size, check_hash=self.check_hash, ) b2-sdk-python-1.17.3/b2sdk/transfer/inbound/downloaded_file.py000066400000000000000000000142461426424117700242030ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/inbound/downloaded_file.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import io import logging from typing import Optional, Tuple, TYPE_CHECKING from requests.models import Response from ...encryption.setting import EncryptionSetting from ...file_version import DownloadVersion from ...progress import AbstractProgressListener from ...stream.progress import WritingStreamWithProgress from b2sdk.exception import ( ChecksumMismatch, TruncatedOutput, ) from b2sdk.utils import set_file_mtime if TYPE_CHECKING: from .download_manager import DownloadManager logger = logging.getLogger(__name__) class MtimeUpdatedFile(io.IOBase): """ Helper class that facilitates updating a files mod_time after closing. Usage: .. code-block: python downloaded_file = bucket.download_file_by_id('b2_file_id') with MtimeUpdatedFile('some_local_path', mod_time_millis=downloaded_file.download_version.mod_time_millis) as file: downloaded_file.save(file) # 'some_local_path' has the mod_time set according to metadata in B2 """ def __init__(self, path_, mod_time_millis: int, mode='wb+', buffering=None): self.path_ = path_ self.mode = mode self.buffering = buffering if buffering is not None else -1 self.mod_time_to_set = mod_time_millis self.file = None def write(self, value): """ This method is overwritten (monkey-patched) in __enter__ for performance reasons """ raise NotImplementedError def read(self, *a): """ This method is overwritten (monkey-patched) in __enter__ for performance reasons """ raise NotImplementedError def seek(self, offset, whence=0): return self.file.seek(offset, whence) def tell(self): return self.file.tell() def __enter__(self): self.file = open(self.path_, self.mode, buffering=self.buffering) self.write = self.file.write self.read = self.file.read return self def __exit__(self, exc_type, exc_val, exc_tb): self.file.close() set_file_mtime(self.path_, self.mod_time_to_set) class DownloadedFile: """ Result of a successful download initialization. Holds information about file's metadata and allows to perform the download. """ def __init__( self, download_version: DownloadVersion, download_manager: 'DownloadManager', range_: Optional[Tuple[int, int]], response: Response, encryption: Optional[EncryptionSetting], progress_listener: AbstractProgressListener, write_buffer_size=None, check_hash=True, ): self.download_version = download_version self.download_manager = download_manager self.range_ = range_ self.response = response self.encryption = encryption self.progress_listener = progress_listener self.download_strategy = None self.write_buffer_size = write_buffer_size self.check_hash = check_hash def _validate_download(self, bytes_read, actual_sha1): if self.download_version.content_encoding is not None and self.download_version.api.api_config.decode_content: return if self.range_ is None: if bytes_read != self.download_version.content_length: raise TruncatedOutput(bytes_read, self.download_version.content_length) if ( self.check_hash and self.download_version.content_sha1 != 'none' and actual_sha1 != self.download_version.content_sha1 ): raise ChecksumMismatch( checksum_type='sha1', expected=self.download_version.content_sha1, actual=actual_sha1, ) else: desired_length = self.range_[1] - self.range_[0] + 1 if bytes_read != desired_length: raise TruncatedOutput(bytes_read, desired_length) def save(self, file, allow_seeking=True): """ Read data from B2 cloud and write it to a file-like object :param file: a file-like object :param allow_seeking: if False, download strategies that rely on seeking to write data (parallel strategies) will be discarded. """ if self.progress_listener: file = WritingStreamWithProgress(file, self.progress_listener) if self.range_ is not None: total_bytes = self.range_[1] - self.range_[0] + 1 else: total_bytes = self.download_version.content_length self.progress_listener.set_total_bytes(total_bytes) for strategy in self.download_manager.strategies: if strategy.is_suitable(self.download_version, allow_seeking): break else: raise ValueError('no strategy suitable for download was found!') self.download_strategy = strategy bytes_read, actual_sha1 = strategy.download( file, response=self.response, download_version=self.download_version, session=self.download_manager.services.session, encryption=self.encryption, ) self._validate_download(bytes_read, actual_sha1) def save_to(self, path_, mode='wb+', allow_seeking=True): """ Open a local file and write data from B2 cloud to it, also update the mod_time. :param path_: path to file to be opened :param mode: mode in which the file should be opened :param allow_seeking: if False, download strategies that rely on seeking to write data (parallel strategies) will be discarded. """ with MtimeUpdatedFile( path_, mod_time_millis=self.download_version.mod_time_millis, mode=mode, buffering=self.write_buffer_size, ) as file: self.save(file, allow_seeking=allow_seeking) b2-sdk-python-1.17.3/b2sdk/transfer/inbound/downloader/000077500000000000000000000000001426424117700226415ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/transfer/inbound/downloader/__init__.py000066400000000000000000000004701426424117700247530ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/inbound/downloader/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/transfer/inbound/downloader/abstract.py000066400000000000000000000074441426424117700250270ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/inbound/downloader/abstract.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import hashlib from abc import abstractmethod from concurrent.futures import ThreadPoolExecutor from io import IOBase from typing import Optional from requests.models import Response from b2sdk.file_version import DownloadVersion from b2sdk.session import B2Session from b2sdk.utils import B2TraceMetaAbstract from b2sdk.utils.range_ import Range from b2sdk.encryption.setting import EncryptionSetting class EmptyHasher: def __init__(self, *args, **kwargs): pass def update(self, data): pass def digest(self): return b'' def hexdigest(self): return '' def copy(self): return self class AbstractDownloader(metaclass=B2TraceMetaAbstract): REQUIRES_SEEKING = True DEFAULT_THREAD_POOL_CLASS = staticmethod(ThreadPoolExecutor) DEFAULT_ALIGN_FACTOR = 4096 def __init__( self, thread_pool: Optional[ThreadPoolExecutor] = None, force_chunk_size: Optional[int] = None, min_chunk_size: Optional[int] = None, max_chunk_size: Optional[int] = None, align_factor: Optional[int] = None, check_hash: bool = True, **kwargs ): align_factor = align_factor or self.DEFAULT_ALIGN_FACTOR assert force_chunk_size is not None or ( min_chunk_size is not None and max_chunk_size is not None and 0 < min_chunk_size <= max_chunk_size and max_chunk_size >= align_factor ) self._min_chunk_size = min_chunk_size self._max_chunk_size = max_chunk_size self._forced_chunk_size = force_chunk_size self._align_factor = align_factor self._check_hash = check_hash self._thread_pool = thread_pool if thread_pool is not None \ else self.DEFAULT_THREAD_POOL_CLASS() super().__init__(**kwargs) def _get_hasher(self): if self._check_hash: return hashlib.sha1() return EmptyHasher() def _get_chunk_size(self, content_length: Optional[int]): if self._forced_chunk_size is not None: return self._forced_chunk_size ideal = max(content_length // 1000, self._align_factor) non_aligned = min(max(ideal, self._min_chunk_size), self._max_chunk_size) aligned = non_aligned // self._align_factor * self._align_factor return aligned @classmethod def _get_remote_range(cls, response: Response, download_version: DownloadVersion): """ Get a range from response or original request (as appropriate). :param response: requests.Response of initial request :param download_version: b2sdk.v2.DownloadVersion :return: a range object """ if 'Range' in response.request.headers: return Range.from_header(response.request.headers['Range']) return download_version.range_ def is_suitable(self, download_version: DownloadVersion, allow_seeking: bool): """ Analyze download_version (possibly against options passed earlier to constructor to find out whether the given download request should be handled by this downloader). """ if self.REQUIRES_SEEKING and not allow_seeking: return False return True @abstractmethod def download( self, file: IOBase, response: Response, download_version: DownloadVersion, session: B2Session, encryption: Optional[EncryptionSetting] = None, ): """ @returns (bytes_read, actual_sha1) """ pass b2-sdk-python-1.17.3/b2sdk/transfer/inbound/downloader/parallel.py000066400000000000000000000347551426424117700250250ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/inbound/downloader/parallel.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from concurrent import futures from io import IOBase from typing import Optional import logging import queue import threading from requests.models import Response from .abstract import AbstractDownloader from b2sdk.encryption.setting import EncryptionSetting from b2sdk.file_version import DownloadVersion from b2sdk.session import B2Session from b2sdk.utils.range_ import Range logger = logging.getLogger(__name__) class ParallelDownloader(AbstractDownloader): # situations to consider: # # local file start local file end # | | # | | # | write range start write range end | # | | | | # v v v v # ####################################################################### # | | | | | | # \ / \ / \ / \ / \ / # part 1 part 2 part 3 part 4 part 5 # / \ / \ / \ / \ / \ # | | | | | | # ####################################################################### # ^ ^ # | | # cloud file start cloud file end # FINISH_HASHING_BUFFER_SIZE = 1024**2 def __init__(self, min_part_size: int, max_streams: Optional[int] = None, **kwargs): """ :param max_streams: maximum number of simultaneous streams :param min_part_size: minimum amount of data a single stream will retrieve, in bytes """ super().__init__(**kwargs) self.max_streams = max_streams self.min_part_size = min_part_size def is_suitable(self, download_version: DownloadVersion, allow_seeking: bool): if not super().is_suitable(download_version, allow_seeking): return False return self._get_number_of_streams( download_version.content_length ) >= 2 and download_version.content_length >= 2 * self.min_part_size def _get_number_of_streams(self, content_length): num_streams = content_length // self.min_part_size if self.max_streams is not None: num_streams = min(num_streams, self.max_streams) else: max_threadpool_workers = getattr(self._thread_pool, '_max_workers', None) if max_threadpool_workers is not None: num_streams = min(num_streams, max_threadpool_workers) return num_streams def download( self, file: IOBase, response: Response, download_version: DownloadVersion, session: B2Session, encryption: Optional[EncryptionSetting] = None, ): """ Download a file from given url using parallel download sessions and stores it in the given download_destination. """ remote_range = self._get_remote_range(response, download_version) actual_size = remote_range.size() start_file_position = file.tell() parts_to_download = list( gen_parts( remote_range, Range(start_file_position, start_file_position + actual_size - 1), part_count=self._get_number_of_streams(download_version.content_length), ) ) first_part = parts_to_download[0] hasher = self._get_hasher() with WriterThread(file, max_queue_depth=len(parts_to_download) * 2) as writer: self._get_parts( response, session, writer, hasher, first_part, parts_to_download[1:], self._get_chunk_size(actual_size), encryption=encryption, ) bytes_written = writer.total # At this point the hasher already consumed the data until the end of first stream. # Consume the rest of the file to complete the hashing process if self._check_hash: # we skip hashing if we would not check it - hasher object is actually a EmptyHasher instance # but we avoid here reading whole file (except for the first part) from disk again self._finish_hashing(first_part, file, hasher, download_version.content_length) return bytes_written, hasher.hexdigest() def _finish_hashing(self, first_part, file, hasher, content_length): end_of_first_part = first_part.local_range.end + 1 file.seek(end_of_first_part) file_read = file.read last_offset = first_part.local_range.start + content_length current_offset = end_of_first_part stop = False while 1: data = file_read(self.FINISH_HASHING_BUFFER_SIZE) if not data: break if current_offset + len(data) >= last_offset: to_hash = data[:last_offset - current_offset] stop = True else: to_hash = data hasher.update(data) current_offset += len(to_hash) if stop: break def _get_parts( self, response, session, writer, hasher, first_part, parts_to_download, chunk_size, encryption ): stream = self._thread_pool.submit( download_first_part, response, hasher, session, writer, first_part, chunk_size, encryption=encryption, ) streams = [stream] for part in parts_to_download: stream = self._thread_pool.submit( download_non_first_part, response.request.url, session, writer, part, chunk_size, encryption=encryption, ) streams.append(stream) futures.wait(streams) class WriterThread(threading.Thread): """ A thread responsible for keeping a queue of data chunks to write to a file-like object and for actually writing them down. Since a single thread is responsible for synchronization of the writes, we avoid a lot of issues between userspace and kernelspace that would normally require flushing buffers between the switches of the writer. That would kill performance and not synchronizing would cause data corruption (probably we'd end up with a file with unexpected blocks of zeros preceding the range of the writer that comes second and writes further into the file). The object of this class is also responsible for backpressure: if items are added to the queue faster than they can be written (see GCP VMs with standard PD storage with faster CPU and network than local storage, https://github.com/Backblaze/B2_Command_Line_Tool/issues/595), then ``obj.queue.put(item)`` will block, slowing down the producer. The recommended minimum value of ``max_queue_depth`` is equal to the amount of producer threads, so that if all producers submit a part at the exact same time (right after network issue, for example, or just after starting the read), they can continue their work without blocking. The writer should be able to store at least one data chunk before a new one is retrieved, but it is not guaranteed. Therefore, the recommended value of ``max_queue_depth`` is higher - a double of the amount of producers, so that spikes on either end (many producers submit at the same time / consumer has a latency spike) can be accommodated without sacrificing performance. Please note that a size of the chunk and the queue depth impact the memory footprint. In a default setting as of writing this, that might be 10 downloads, 8 producers, 1MB buffers, 2 buffers each = 8*2*10 = 160 MB (+ python buffers, operating system etc). """ def __init__(self, file, max_queue_depth): self.file = file self.queue = queue.Queue(max_queue_depth) self.total = 0 super(WriterThread, self).__init__() def run(self): file = self.file queue_get = self.queue.get while 1: shutdown, offset, data = queue_get() if shutdown: break file.seek(offset) file.write(data) self.total += len(data) def __enter__(self): self.start() return self def __exit__(self, exc_type, exc_val, exc_tb): self.queue.put((True, None, None)) self.join() def download_first_part( response: Response, hasher, session: B2Session, writer: WriterThread, first_part: 'PartToDownload', chunk_size: int, encryption: Optional[EncryptionSetting] = None, ) -> None: """ :param response: response of the original GET call :param hasher: hasher object to feed to as the stream is written :param session: B2 API session :param writer: thread responsible for writing downloaded data :param first_part: definition of the part to be downloaded :param chunk_size: size (in bytes) of read data chunks :param encryption: encryption mode, algorithm and key """ writer_queue_put = writer.queue.put hasher_update = hasher.update first_offset = first_part.local_range.start last_offset = first_part.local_range.end + 1 actual_part_size = first_part.local_range.size() starting_cloud_range = first_part.cloud_range bytes_read = 0 stop = False for data in response.iter_content(chunk_size=chunk_size): if first_offset + bytes_read + len(data) >= last_offset: to_write = data[:last_offset - bytes_read] stop = True else: to_write = data writer_queue_put((False, first_offset + bytes_read, to_write)) hasher_update(to_write) bytes_read += len(to_write) if stop: break # since we got everything we need from original response, close the socket and free the buffer # to avoid a timeout exception during hashing and other trouble response.close() url = response.request.url tries_left = 5 - 1 # this is hardcoded because we are going to replace the entire retry interface soon, so we'll avoid deprecation here and keep it private while tries_left and bytes_read < actual_part_size: cloud_range = starting_cloud_range.subrange( bytes_read, actual_part_size - 1 ) # first attempt was for the whole file, but retries are bound correctly logger.debug( 'download attempts remaining: %i, bytes read already: %i. Getting range %s now.', tries_left, bytes_read, cloud_range ) with session.download_file_from_url( url, cloud_range.as_tuple(), encryption=encryption, ) as response: for to_write in response.iter_content(chunk_size=chunk_size): writer_queue_put((False, first_offset + bytes_read, to_write)) hasher_update(to_write) bytes_read += len(to_write) tries_left -= 1 def download_non_first_part( url: str, session: B2Session, writer: WriterThread, part_to_download: 'PartToDownload', chunk_size: int, encryption: Optional[EncryptionSetting] = None, ) -> None: """ :param url: download URL :param session: B2 API session :param writer: thread responsible for writing downloaded data :param part_to_download: definition of the part to be downloaded :param chunk_size: size (in bytes) of read data chunks :param encryption: encryption mode, algorithm and key """ writer_queue_put = writer.queue.put start_range = part_to_download.local_range.start actual_part_size = part_to_download.local_range.size() bytes_read = 0 starting_cloud_range = part_to_download.cloud_range retries_left = 5 # this is hardcoded because we are going to replace the entire retry interface soon, so we'll avoid deprecation here and keep it private while retries_left and bytes_read < actual_part_size: cloud_range = starting_cloud_range.subrange(bytes_read, actual_part_size - 1) logger.debug( 'download attempts remaining: %i, bytes read already: %i. Getting range %s now.', retries_left, bytes_read, cloud_range ) with session.download_file_from_url( url, cloud_range.as_tuple(), encryption=encryption, ) as response: for to_write in response.iter_content(chunk_size=chunk_size): writer_queue_put((False, start_range + bytes_read, to_write)) bytes_read += len(to_write) retries_left -= 1 class PartToDownload: """ Hold the range of a file to download, and the range of the local file where it should be stored. """ def __init__(self, cloud_range, local_range): self.cloud_range = cloud_range self.local_range = local_range def __repr__(self): return 'PartToDownload(%s, %s)' % (self.cloud_range, self.local_range) def gen_parts(cloud_range, local_range, part_count): """ Generate a sequence of PartToDownload to download a large file as a collection of parts. """ assert cloud_range.size() == local_range.size(), (cloud_range.size(), local_range.size()) assert 0 < part_count <= cloud_range.size() offset = 0 remaining_size = cloud_range.size() for i in range(part_count): # This rounds down, so if the parts aren't all the same size, # the smaller parts will come first. this_part_size = remaining_size // (part_count - i) part = PartToDownload( cloud_range.subrange(offset, offset + this_part_size - 1), local_range.subrange(offset, offset + this_part_size - 1), ) logger.debug('created part to download: %s', part) yield part offset += this_part_size remaining_size -= this_part_size b2-sdk-python-1.17.3/b2sdk/transfer/inbound/downloader/simple.py000066400000000000000000000066241426424117700245140ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/inbound/downloader/simple.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from io import IOBase from typing import Optional import logging from requests.models import Response from .abstract import AbstractDownloader from b2sdk.file_version import DownloadVersion from b2sdk.encryption.setting import EncryptionSetting from b2sdk.session import B2Session logger = logging.getLogger(__name__) class SimpleDownloader(AbstractDownloader): REQUIRES_SEEKING = False def _download( self, file: IOBase, response: Response, download_version: DownloadVersion, session: B2Session, encryption: Optional[EncryptionSetting] = None, ): actual_size = self._get_remote_range(response, download_version).size() chunk_size = self._get_chunk_size(actual_size) digest = self._get_hasher() bytes_read = 0 for data in response.iter_content(chunk_size=chunk_size): file.write(data) digest.update(data) bytes_read += len(data) assert actual_size >= 1 # code below does `actual_size - 1`, but it should never reach that part with an empty file # now, normally bytes_read == download_version.content_length, but sometimes there is a timeout # or something and the server closes connection, while neither tcp or http have a problem # with the truncated output, so we detect it here and try to continue num_tries = 5 # this is hardcoded because we are going to replace the entire retry interface soon, so we'll avoid deprecation here and keep it private retries_left = num_tries - 1 while retries_left and bytes_read < download_version.content_length: new_range = self._get_remote_range( response, download_version, ).subrange(bytes_read, actual_size - 1) # original response is not closed at this point yet, as another layer is responsible for closing it, so a new socket might be allocated, # but this is a very rare case and so it is not worth the optimization logger.debug( 're-download attempts remaining: %i, bytes read already: %i. Getting range %s now.', retries_left, bytes_read, new_range ) with session.download_file_from_url( response.request.url, new_range.as_tuple(), encryption=encryption, ) as followup_response: for data in followup_response.iter_content( chunk_size=self._get_chunk_size(actual_size) ): file.write(data) digest.update(data) bytes_read += len(data) retries_left -= 1 return bytes_read, digest.hexdigest() def download( self, file: IOBase, response: Response, download_version: DownloadVersion, session: B2Session, encryption: Optional[EncryptionSetting] = None, ): future = self._thread_pool.submit( self._download, file, response, download_version, session, encryption ) return future.result() b2-sdk-python-1.17.3/b2sdk/transfer/outbound/000077500000000000000000000000001426424117700207045ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/transfer/outbound/__init__.py000066400000000000000000000004561426424117700230220ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/outbound/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/transfer/outbound/copy_manager.py000066400000000000000000000243371426424117700237330ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/outbound/copy_manager.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging from typing import Optional from b2sdk.encryption.setting import EncryptionMode, EncryptionSetting from b2sdk.http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME from b2sdk.exception import AlreadyFailed, CopyArgumentsMismatch, SSECKeyIdMismatchInCopy from b2sdk.file_lock import FileRetentionSetting, LegalHold from b2sdk.raw_api import MetadataDirectiveMode from b2sdk.transfer.transfer_manager import TransferManager from b2sdk.utils.thread_pool import ThreadPoolMixin logger = logging.getLogger(__name__) class CopyManager(TransferManager, ThreadPoolMixin): """ Handle complex actions around server side copy to free raw_api from that responsibility. """ MAX_LARGE_FILE_SIZE = 10 * 1000 * 1000 * 1000 * 1000 # 10 TB @property def account_info(self): return self.services.session.account_info def copy_file( self, copy_source, file_name, content_type, file_info, destination_bucket_id, progress_listener, destination_encryption: Optional[EncryptionSetting] = None, source_encryption: Optional[EncryptionSetting] = None, legal_hold: Optional[LegalHold] = None, file_retention: Optional[FileRetentionSetting] = None, ): # Run small copies in the same thread pool as large file copies, # so that they share resources during a sync. return self._thread_pool.submit( self._copy_small_file, copy_source, file_name, content_type=content_type, file_info=file_info, destination_bucket_id=destination_bucket_id, progress_listener=progress_listener, destination_encryption=destination_encryption, source_encryption=source_encryption, legal_hold=legal_hold, file_retention=file_retention, ) def copy_part( self, large_file_id, part_copy_source, part_number, large_file_upload_state, finished_parts=None, destination_encryption: Optional[EncryptionSetting] = None, source_encryption: Optional[EncryptionSetting] = None, ): return self._thread_pool.submit( self._copy_part, large_file_id, part_copy_source, part_number, large_file_upload_state, finished_parts=finished_parts, destination_encryption=destination_encryption, source_encryption=source_encryption, ) def _copy_part( self, large_file_id, part_copy_source, part_number, large_file_upload_state, finished_parts, destination_encryption: Optional[EncryptionSetting], source_encryption: Optional[EncryptionSetting], ): """ Copy a file part to started large file. :param :param str bucket_id: a bucket ID :param large_file_id: a large file ID :param b2sdk.v2.CopySource part_copy_source: copy source that represents a range (not necessarily a whole file) :param b2sdk.v2.LargeFileUploadState large_file_upload_state: state object for progress reporting on large file upload :param dict,None finished_parts: dictionary of known finished parts, keys are part numbers, values are instances of :class:`~b2sdk.v2.Part` :param b2sdk.v2.EncryptionSetting destination_encryption: encryption settings for the destination (``None`` if unknown) :param b2sdk.v2.EncryptionSetting source_encryption: encryption settings for the source (``None`` if unknown) """ # b2_copy_part doesn't need SSE-B2. Large file encryption is decided on b2_start_large_file. if destination_encryption is not None and destination_encryption.mode == EncryptionMode.SSE_B2: destination_encryption = None # Check if this part was uploaded before if finished_parts is not None and part_number in finished_parts: # Report this part finished part = finished_parts[part_number] large_file_upload_state.update_part_bytes(part.content_length) # Return SHA1 hash return {'contentSha1': part.content_sha1} # if another part has already had an error there's no point in # uploading this part if large_file_upload_state.has_error(): raise AlreadyFailed(large_file_upload_state.get_error_message()) response = self.services.session.copy_part( part_copy_source.file_id, large_file_id, part_number, bytes_range=part_copy_source.get_bytes_range(), destination_server_side_encryption=destination_encryption, source_server_side_encryption=source_encryption, ) large_file_upload_state.update_part_bytes(response['contentLength']) return response def _copy_small_file( self, copy_source, file_name, content_type, file_info, destination_bucket_id, progress_listener, destination_encryption: Optional[EncryptionSetting], source_encryption: Optional[EncryptionSetting], legal_hold: Optional[LegalHold] = None, file_retention: Optional[FileRetentionSetting] = None, ): with progress_listener: progress_listener.set_total_bytes(copy_source.get_content_length() or 0) bytes_range = copy_source.get_bytes_range() if content_type is None: if file_info is not None: raise CopyArgumentsMismatch( 'File info can be set only when content type is set' ) metadata_directive = MetadataDirectiveMode.COPY else: if file_info is None: raise CopyArgumentsMismatch( 'File info can be not set only when content type is not set' ) metadata_directive = MetadataDirectiveMode.REPLACE metadata_directive, file_info, content_type = self.establish_sse_c_file_metadata( metadata_directive=metadata_directive, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=destination_encryption, source_server_side_encryption=source_encryption, source_file_info=copy_source.source_file_info, source_content_type=copy_source.source_content_type, ) response = self.services.session.copy_file( copy_source.file_id, file_name, bytes_range=bytes_range, metadata_directive=metadata_directive, content_type=content_type, file_info=file_info, destination_bucket_id=destination_bucket_id, destination_server_side_encryption=destination_encryption, source_server_side_encryption=source_encryption, legal_hold=legal_hold, file_retention=file_retention, ) file_version = self.services.api.file_version_factory.from_api_response(response) if progress_listener is not None: progress_listener.bytes_completed(file_version.size) return file_version @classmethod def establish_sse_c_file_metadata( cls, metadata_directive: MetadataDirectiveMode, destination_file_info: Optional[dict], destination_content_type: Optional[str], destination_server_side_encryption: Optional[EncryptionSetting], source_server_side_encryption: Optional[EncryptionSetting], source_file_info: Optional[dict], source_content_type: Optional[str], ): assert metadata_directive in (MetadataDirectiveMode.REPLACE, MetadataDirectiveMode.COPY) if metadata_directive == MetadataDirectiveMode.REPLACE: if destination_server_side_encryption: destination_file_info = destination_server_side_encryption.add_key_id_to_file_info( destination_file_info ) return metadata_directive, destination_file_info, destination_content_type source_key_id = None destination_key_id = None if destination_server_side_encryption is not None and destination_server_side_encryption.key is not None and \ destination_server_side_encryption.key.key_id is not None: destination_key_id = destination_server_side_encryption.key.key_id if source_server_side_encryption is not None and source_server_side_encryption.key is not None and \ source_server_side_encryption.key.key_id is not None: source_key_id = source_server_side_encryption.key.key_id if source_key_id == destination_key_id: return metadata_directive, destination_file_info, destination_content_type if source_file_info is None or source_content_type is None: raise SSECKeyIdMismatchInCopy( 'attempting to copy file using %s without providing source_file_info ' 'and source_content_type for differing sse_c_key_ids: source="%s", ' 'destination="%s"' % (MetadataDirectiveMode.COPY, source_key_id, destination_key_id) ) destination_file_info = source_file_info.copy() destination_file_info.pop(SSE_C_KEY_ID_FILE_INFO_KEY_NAME, None) if destination_server_side_encryption: destination_file_info = destination_server_side_encryption.add_key_id_to_file_info( destination_file_info ) destination_content_type = source_content_type return MetadataDirectiveMode.REPLACE, destination_file_info, destination_content_type b2-sdk-python-1.17.3/b2sdk/transfer/outbound/copy_source.py000066400000000000000000000053501426424117700236130ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/outbound/copy_source.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional from b2sdk.encryption.setting import EncryptionSetting from b2sdk.transfer.outbound.outbound_source import OutboundTransferSource class CopySource(OutboundTransferSource): def __init__( self, file_id, offset=0, length=None, encryption: Optional[EncryptionSetting] = None, source_file_info=None, source_content_type=None, ): if not length and offset > 0: raise ValueError('Cannot copy with non zero offset and unknown or zero length') self.file_id = file_id self.length = length self.offset = offset self.encryption = encryption self.source_file_info = source_file_info self.source_content_type = source_content_type def __repr__(self): return ( '<{classname} file_id={file_id} offset={offset} length={length} id={id}, encryption={encryption},' 'source_content_type={source_content_type}>, source_file_info={source_file_info}' ).format( classname=self.__class__.__name__, file_id=self.file_id, offset=self.offset, length=self.length, id=id(self), encryption=self.encryption, source_content_type=self.source_content_type, source_file_info=self.source_file_info, ) def get_content_length(self): return self.length def is_upload(self): return False def is_copy(self): return True def get_bytes_range(self): if not self.length: if self.offset > 0: # auto mode should get file info and create correct copy source (with length) raise ValueError( 'cannot return bytes range for non zero offset and unknown or zero length' ) return None return self.offset, self.offset + self.length - 1 def get_copy_source_range(self, relative_offset, range_length): if self.length is not None and range_length + relative_offset > self.length: raise ValueError('Range length overflow source length') range_offset = self.offset + relative_offset return self.__class__( self.file_id, range_offset, range_length, encryption=self.encryption, source_file_info=self.source_file_info, source_content_type=self.source_content_type ) b2-sdk-python-1.17.3/b2sdk/transfer/outbound/large_file_upload_state.py000066400000000000000000000040251426424117700261140ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/outbound/large_file_upload_state.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import threading class LargeFileUploadState: """ Track the status of uploading a large file, accepting updates from the tasks that upload each of the parts. The aggregated progress is passed on to a ProgressListener that reports the progress for the file as a whole. This class is THREAD SAFE. """ def __init__(self, file_progress_listener): """ :param b2sdk.v2.AbstractProgressListener file_progress_listener: a progress listener object to use. Use :py:class:`b2sdk.v2.DoNothingProgressListener` to disable. """ self.lock = threading.RLock() self.error_message = None self.file_progress_listener = file_progress_listener self.part_number_to_part_state = {} self.bytes_completed = 0 def set_error(self, message): """ Set an error message. :param str message: an error message """ with self.lock: self.error_message = message def has_error(self): """ Check whether an error occurred. :rtype: bool """ with self.lock: return self.error_message is not None def get_error_message(self): """ Fetche an error message. :return: an error message :rtype: str """ with self.lock: assert self.has_error() return self.error_message def update_part_bytes(self, bytes_delta): """ Update listener progress info. :param int bytes_delta: number of bytes to increase a progress for """ with self.lock: self.bytes_completed += bytes_delta self.file_progress_listener.bytes_completed(self.bytes_completed) b2-sdk-python-1.17.3/b2sdk/transfer/outbound/outbound_source.py000066400000000000000000000022471426424117700245020ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/outbound/outbound_source.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import ABCMeta, abstractmethod class OutboundTransferSource(metaclass=ABCMeta): """ Abstract class for defining outbound transfer sources. Supported outbound transfer sources are: * :class:`b2sdk.v2.CopySource` * :class:`b2sdk.v2.UploadSourceBytes` * :class:`b2sdk.v2.UploadSourceLocalFile` * :class:`b2sdk.v2.UploadSourceLocalFileRange` * :class:`b2sdk.v2.UploadSourceStream` * :class:`b2sdk.v2.UploadSourceStreamRange` """ @abstractmethod def get_content_length(self): """ Return the number of bytes of data in the file. """ @abstractmethod def is_upload(self): """ Return if outbound source is an upload source. :rtype bool: """ @abstractmethod def is_copy(self): """ Return if outbound source is a copy source. :rtype bool: """ b2-sdk-python-1.17.3/b2sdk/transfer/outbound/progress_reporter.py000066400000000000000000000026651426424117700250550ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/outbound/progress_reporter.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.progress import AbstractProgressListener class PartProgressReporter(AbstractProgressListener): """ An adapter that listens to the progress of upload a part and gives the information to a :py:class:`b2sdk.bucket.LargeFileUploadState`. Accepts absolute bytes_completed from the uploader, and reports deltas to the :py:class:`b2sdk.bucket.LargeFileUploadState`. The bytes_completed for the part will drop back to 0 on a retry, which will result in a negative delta. """ def __init__(self, large_file_upload_state, *args, **kwargs): """ :param b2sdk.bucket.LargeFileUploadState large_file_upload_state: object to relay the progress to """ super(PartProgressReporter, self).__init__(*args, **kwargs) self.large_file_upload_state = large_file_upload_state self.prev_byte_count = 0 def bytes_completed(self, byte_count): self.large_file_upload_state.update_part_bytes(byte_count - self.prev_byte_count) self.prev_byte_count = byte_count def close(self): pass def set_total_bytes(self, total_byte_count): pass b2-sdk-python-1.17.3/b2sdk/transfer/outbound/upload_manager.py000066400000000000000000000210371426424117700242370ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/outbound/upload_manager.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging from typing import Optional from b2sdk.encryption.setting import EncryptionMode, EncryptionSetting from b2sdk.exception import ( AlreadyFailed, B2Error, MaxRetriesExceeded, ) from b2sdk.file_lock import FileRetentionSetting, LegalHold from b2sdk.stream.progress import ReadingStreamWithProgress from b2sdk.stream.hashing import StreamWithHash from b2sdk.http_constants import HEX_DIGITS_AT_END from .progress_reporter import PartProgressReporter from ..transfer_manager import TransferManager from ...utils.thread_pool import ThreadPoolMixin logger = logging.getLogger(__name__) class UploadManager(TransferManager, ThreadPoolMixin): """ Handle complex actions around uploads to free raw_api from that responsibility. """ MAX_UPLOAD_ATTEMPTS = 5 @property def account_info(self): return self.services.session.account_info def upload_file( self, bucket_id, upload_source, file_name, content_type, file_info, progress_listener, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): f = self._thread_pool.submit( self._upload_small_file, bucket_id, upload_source, file_name, content_type, file_info, progress_listener, encryption, file_retention, legal_hold, ) return f def upload_part( self, bucket_id, file_id, part_upload_source, part_number, large_file_upload_state, finished_parts=None, encryption: EncryptionSetting = None, ): f = self._thread_pool.submit( self._upload_part, bucket_id, file_id, part_upload_source, part_number, large_file_upload_state, finished_parts, encryption, ) return f def _upload_part( self, bucket_id, file_id, part_upload_source, part_number, large_file_upload_state, finished_parts, encryption: EncryptionSetting, ): """ Upload a file part to started large file. :param :param str bucket_id: a bucket ID :param file_id: a large file ID :param b2sdk.v2.AbstractUploadSource part_upload_source: upload source that reads only required range :param b2sdk.v2.LargeFileUploadState large_file_upload_state: state object for progress reporting on large file upload :param dict,None finished_parts: dictionary of known finished parts, keys are part numbers, values are instances of :class:`~b2sdk.v2.Part` :param b2sdk.v2.EncryptionSetting encryption: encryption setting (``None`` if unknown) """ # b2_upload_part doesn't need SSE-B2. Large file encryption is decided on b2_start_large_file. if encryption is not None and encryption.mode == EncryptionMode.SSE_B2: encryption = None # Check if this part was uploaded before if finished_parts is not None and part_number in finished_parts: # Report this part finished part = finished_parts[part_number] large_file_upload_state.update_part_bytes(part_upload_source.get_content_length()) # Return SHA1 hash return {'contentSha1': part.content_sha1} # Set up a progress listener part_progress_listener = PartProgressReporter(large_file_upload_state) # Retry the upload as needed exception_list = [] for _ in range(self.MAX_UPLOAD_ATTEMPTS): # if another part has already had an error there's no point in # uploading this part if large_file_upload_state.has_error(): raise AlreadyFailed(large_file_upload_state.get_error_message()) try: with part_upload_source.open() as part_stream: content_length = part_upload_source.get_content_length() input_stream = ReadingStreamWithProgress( part_stream, part_progress_listener, length=content_length ) if part_upload_source.is_sha1_known(): content_sha1 = part_upload_source.get_content_sha1() else: input_stream = StreamWithHash(input_stream, stream_length=content_length) content_sha1 = HEX_DIGITS_AT_END # it is important that `len()` works on `input_stream` response = self.services.session.upload_part( file_id, part_number, len(input_stream), content_sha1, input_stream, server_side_encryption=encryption, # todo: client side encryption ) if content_sha1 == HEX_DIGITS_AT_END: content_sha1 = input_stream.hash assert content_sha1 == response['contentSha1'] return response except B2Error as e: if not e.should_retry_upload(): raise exception_list.append(e) self.account_info.clear_bucket_upload_data(bucket_id) large_file_upload_state.set_error(str(exception_list[-1])) raise MaxRetriesExceeded(self.MAX_UPLOAD_ATTEMPTS, exception_list) def _upload_small_file( self, bucket_id, upload_source, file_name, content_type, file_info, progress_listener, encryption: Optional[EncryptionSetting] = None, file_retention: Optional[FileRetentionSetting] = None, legal_hold: Optional[LegalHold] = None, ): content_length = upload_source.get_content_length() exception_info_list = [] progress_listener.set_total_bytes(content_length) with progress_listener: for _ in range(self.MAX_UPLOAD_ATTEMPTS): try: with upload_source.open() as file: input_stream = ReadingStreamWithProgress( file, progress_listener, length=content_length ) if upload_source.is_sha1_known(): content_sha1 = upload_source.get_content_sha1() else: input_stream = StreamWithHash( input_stream, stream_length=content_length ) content_sha1 = HEX_DIGITS_AT_END # it is important that `len()` works on `input_stream` response = self.services.session.upload_file( bucket_id, file_name, len(input_stream), content_type, content_sha1, file_info, input_stream, server_side_encryption=encryption, # todo: client side encryption file_retention=file_retention, legal_hold=legal_hold, ) if content_sha1 == HEX_DIGITS_AT_END: content_sha1 = input_stream.hash assert content_sha1 == 'do_not_verify' or content_sha1 == response[ 'contentSha1'], '%s != %s' % (content_sha1, response['contentSha1']) return self.services.api.file_version_factory.from_api_response(response) except B2Error as e: if not e.should_retry_upload(): raise exception_info_list.append(e) self.account_info.clear_bucket_upload_data(bucket_id) raise MaxRetriesExceeded(self.MAX_UPLOAD_ATTEMPTS, exception_info_list) b2-sdk-python-1.17.3/b2sdk/transfer/outbound/upload_source.py000066400000000000000000000154641426424117700241340ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/outbound/upload_source.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import hashlib import io import os from abc import abstractmethod from b2sdk.exception import InvalidUploadSource from b2sdk.stream.range import RangeOfInputStream, wrap_with_range from b2sdk.transfer.outbound.outbound_source import OutboundTransferSource from b2sdk.utils import hex_sha1_of_stream, hex_sha1_of_unlimited_stream class AbstractUploadSource(OutboundTransferSource): """ The source of data for uploading to b2. """ @abstractmethod def get_content_sha1(self): """ Return a 40-character string containing the hex SHA1 checksum of the data in the file. """ @abstractmethod def open(self): """ Return a binary file-like object from which the data can be read. :return: """ def is_upload(self): return True def is_copy(self): return False def is_sha1_known(self): return False class UploadSourceBytes(AbstractUploadSource): def __init__(self, data_bytes, content_sha1=None): self.data_bytes = data_bytes self.content_sha1 = content_sha1 def __repr__(self): return '<{classname} data={data} id={id}>'.format( classname=self.__class__.__name__, data=str(self.data_bytes[:20]) + '...' if len(self.data_bytes) > 20 else self.data_bytes, id=id(self), ) def get_content_length(self): return len(self.data_bytes) def get_content_sha1(self): if self.content_sha1 is None: self.content_sha1 = hashlib.sha1(self.data_bytes).hexdigest() return self.content_sha1 def open(self): return io.BytesIO(self.data_bytes) def is_sha1_known(self): return self.content_sha1 is not None class UploadSourceLocalFile(AbstractUploadSource): def __init__(self, local_path, content_sha1=None): self.local_path = local_path self.content_length = 0 self.check_path_and_get_size() self.content_sha1 = content_sha1 def check_path_and_get_size(self): if not os.path.isfile(self.local_path): raise InvalidUploadSource(self.local_path) self.content_length = os.path.getsize(self.local_path) def __repr__(self): return ( '<{classname} local_path={local_path} content_length={content_length} ' 'content_sha1={content_sha1} id={id}>' ).format( classname=self.__class__.__name__, local_path=self.local_path, content_length=self.content_length, content_sha1=self.content_sha1, id=id(self), ) def get_content_length(self): return self.content_length def get_content_sha1(self): if self.content_sha1 is None: self.content_sha1 = self._hex_sha1_of_file(self.local_path) return self.content_sha1 def open(self): return io.open(self.local_path, 'rb') def _hex_sha1_of_file(self, local_path): with self.open() as f: return hex_sha1_of_stream(f, self.content_length) def is_sha1_known(self): return self.content_sha1 is not None class UploadSourceLocalFileRange(UploadSourceLocalFile): def __init__(self, local_path, content_sha1=None, offset=0, length=None): super(UploadSourceLocalFileRange, self).__init__(local_path, content_sha1) self.file_size = self.content_length self.offset = offset if length is None: self.content_length = self.file_size - self.offset else: if length + self.offset > self.file_size: raise ValueError('Range length overflow file size') self.content_length = length def __repr__(self): return ( '<{classname} local_path={local_path} offset={offset} ' 'content_length={content_length} content_sha1={content_sha1} id={id}>' ).format( classname=self.__class__.__name__, local_path=self.local_path, offset=self.offset, content_length=self.content_length, content_sha1=self.content_sha1, id=id(self), ) def open(self): fp = super(UploadSourceLocalFileRange, self).open() return wrap_with_range(fp, self.file_size, self.offset, self.content_length) class UploadSourceStream(AbstractUploadSource): def __init__(self, stream_opener, stream_length=None, stream_sha1=None): self.stream_opener = stream_opener self._content_length = stream_length self._content_sha1 = stream_sha1 def __repr__(self): return ( '<{classname} stream_opener={stream_opener} content_length={content_length} ' 'content_sha1={content_sha1} id={id}>' ).format( classname=self.__class__.__name__, stream_opener=repr(self.stream_opener), content_length=self._content_length, content_sha1=self._content_sha1, id=id(self), ) def get_content_length(self): if self._content_length is None: self._set_content_length_and_sha1() return self._content_length def get_content_sha1(self): if self._content_sha1 is None: self._set_content_length_and_sha1() return self._content_sha1 def open(self): return self.stream_opener() def _set_content_length_and_sha1(self): sha1, content_length = hex_sha1_of_unlimited_stream(self.open()) self._content_length = content_length self._content_sha1 = sha1 def is_sha1_known(self): return self._content_sha1 is not None class UploadSourceStreamRange(UploadSourceStream): def __init__(self, stream_opener, offset, stream_length, stream_sha1=None): super(UploadSourceStreamRange, self).__init__( stream_opener, stream_length=stream_length, stream_sha1=stream_sha1, ) self._offset = offset def __repr__(self): return ( '<{classname} stream_opener={stream_opener} offset={offset} ' 'content_length={content_length} content_sha1={content_sha1} id={id}>' ).format( classname=self.__class__.__name__, stream_opener=repr(self.stream_opener), offset=self._offset, content_length=self._content_length, content_sha1=self._content_sha1, id=id(self), ) def open(self): return RangeOfInputStream( super(UploadSourceStreamRange, self).open(), self._offset, self._content_length ) b2-sdk-python-1.17.3/b2sdk/transfer/transfer_manager.py000066400000000000000000000010031426424117700227270ustar00rootroot00000000000000###################################################################### # # File: b2sdk/transfer/transfer_manager.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### class TransferManager: """ Base class for manager classes (copy, upload, download) """ def __init__(self, services, **kwargs): self.services = services super().__init__(**kwargs) b2-sdk-python-1.17.3/b2sdk/utils/000077500000000000000000000000001426424117700163615ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/utils/__init__.py000066400000000000000000000300031426424117700204660ustar00rootroot00000000000000###################################################################### # # File: b2sdk/utils/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import base64 import hashlib import os import platform import re import shutil import tempfile import time import concurrent.futures as futures from decimal import Decimal from urllib.parse import quote, unquote_plus from logfury.v1 import DefaultTraceAbstractMeta, DefaultTraceMeta, limit_trace_arguments, disable_trace, trace_call def b2_url_encode(s): """ URL-encode a unicode string to be sent to B2 in an HTTP header. :param s: a unicode string to encode :type s: str :return: URL-encoded string :rtype: str """ return quote(s.encode('utf-8')) def b2_url_decode(s): """ Decode a Unicode string returned from B2 in an HTTP header. :param s: a unicode string to decode :type s: str :return: a Python unicode string. :rtype: str """ return unquote_plus(s) def choose_part_ranges(content_length, minimum_part_size): """ Return a list of (offset, length) for the parts of a large file. :param content_length: content length value :type content_length: int :param minimum_part_size: a minimum file part size :type minimum_part_size: int :rtype: list """ # If the file is at least twice the minimum part size, we are guaranteed # to be able to break it into multiple parts that are all at least # the minimum part size. assert minimum_part_size * 2 <= content_length # How many parts can we make? part_count = min(content_length // minimum_part_size, 10000) assert 2 <= part_count # All of the parts, except the last, are the same size. The # last one may be bigger. part_size = content_length // part_count last_part_size = content_length - (part_size * (part_count - 1)) assert minimum_part_size <= last_part_size # Make all of the parts except the last parts = [(i * part_size, part_size) for i in range(part_count - 1)] # Add the last part start_of_last = (part_count - 1) * part_size last_part = (start_of_last, content_length - start_of_last) parts.append(last_part) return parts def hex_sha1_of_stream(input_stream, content_length): """ Return the 40-character hex SHA1 checksum of the first content_length bytes in the input stream. :param input_stream: stream object, which exposes read() method :param content_length: expected length of the stream :type content_length: int :rtype: str """ remaining = content_length block_size = 1024 * 1024 digest = hashlib.sha1() while remaining != 0: to_read = min(remaining, block_size) data = input_stream.read(to_read) if len(data) != to_read: raise ValueError( 'content_length(%s) is more than the size of the file' % content_length ) digest.update(data) remaining -= to_read return digest.hexdigest() def hex_sha1_of_unlimited_stream(input_stream, limit=None): block_size = 1024 * 1024 content_length = 0 digest = hashlib.sha1() while True: if limit is not None: to_read = min(limit - content_length, block_size) else: to_read = block_size data = input_stream.read(to_read) data_len = len(data) if data_len > 0: digest.update(data) content_length += data_len if data_len < to_read: return digest.hexdigest(), content_length def hex_sha1_of_file(path_): with open(path_, 'rb') as file: return hex_sha1_of_unlimited_stream(file) def hex_sha1_of_bytes(data: bytes) -> str: """ Return the 40-character hex SHA1 checksum of the data. """ return hashlib.sha1(data).hexdigest() def hex_md5_of_bytes(data: bytes) -> str: """ Return the 32-character hex MD5 checksum of the data. """ return hashlib.md5(data).hexdigest() def md5_of_bytes(data: bytes) -> bytes: """ Return the 16-byte MD5 checksum of the data. """ return hashlib.md5(data).digest() def b64_of_bytes(data: bytes) -> str: """ Return the base64 encoded represtantion of the data. """ return base64.b64encode(data).decode() def validate_b2_file_name(name): """ Raise a ValueError if the name is not a valid B2 file name. :param name: a string to check :type name: str """ if not isinstance(name, str): raise ValueError('file name must be a string, not bytes') name_utf8 = name.encode('utf-8') if len(name_utf8) < 1: raise ValueError('file name too short (0 utf-8 bytes)') if 1000 < len(name_utf8): raise ValueError('file name too long (more than 1000 utf-8 bytes)') if name[0] == '/': raise ValueError("file names must not start with '/'") if name[-1] == '/': raise ValueError("file names must not end with '/'") if '\\' in name: raise ValueError("file names must not contain '\\'") if '//' in name: raise ValueError("file names must not contain '//'") if chr(127) in name: raise ValueError("file names must not contain DEL") if any(250 < len(segment) for segment in name_utf8.split(b'/')): raise ValueError("file names segments (between '/') can be at most 250 utf-8 bytes") def is_file_readable(local_path, reporter=None): """ Check if the local file has read permissions. :param local_path: a file path :type local_path: str :param reporter: reporter object to put errors on :rtype: bool """ if not os.path.exists(local_path): if reporter is not None: reporter.local_access_error(local_path) return False elif not os.access(local_path, os.R_OK): if reporter is not None: reporter.local_permission_error(local_path) return False return True def get_file_mtime(local_path): """ Get modification time of a file in milliseconds. :param local_path: a file path :type local_path: str :rtype: int """ mod_time = os.path.getmtime(local_path) * 1000 return int(mod_time) def set_file_mtime(local_path, mod_time_millis): """ Set modification time of a file in milliseconds. :param local_path: a file path :type local_path: str :param mod_time_millis: time to be set :type mod_time_millis: int """ mod_time = mod_time_millis / 1000.0 # We have to convert it this way to avoid differences when mtime # is read from the local file in the next iterations, and time is fetched # without rounding. # This is caused by floating point arithmetic as POSIX systems # represents mtime as floats and B2 as integers. # E.g. for 1093258377393, it would be converted to 1093258377.393 # which is actually represented by 1093258377.3929998874664306640625. # When we save mtime and read it again, we will end up with 1093258377392. # See #617 for details. mod_time = float(Decimal('%.3f5' % mod_time)) os.utime(local_path, (mod_time, mod_time)) def fix_windows_path_limit(path): """ Prefix paths when running on Windows to overcome 260 character path length limit. See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath :param path: a path to prefix :type path: str :return: a prefixed path :rtype: str """ if platform.system() == 'Windows': if path.startswith('\\\\'): # UNC network path return '\\\\?\\UNC\\' + path[2:] elif os.path.isabs(path): # local absolute path return '\\\\?\\' + path else: # relative path, don't alter return path else: return path class TempDir: """ Context manager that creates and destroys a temporary directory. """ def __enter__(self): """ Return the unicode path to the temp dir. """ dirpath_bytes = tempfile.mkdtemp() self.dirpath = str(dirpath_bytes.replace('\\', '\\\\')) return self.dirpath def __exit__(self, exc_type, exc_val, exc_tb): shutil.rmtree(self.dirpath) return None # do not hide exception def _pick_scale_and_suffix(x): # suffixes for different scales suffixes = ' kMGTP' # We want to use the biggest suffix that makes sense. ref_digits = str(int(x)) index = (len(ref_digits) - 1) // 3 suffix = suffixes[index] if suffix == ' ': suffix = '' scale = 1000**index return (scale, suffix) def format_and_scale_number(x, unit): """ Pick a good scale for representing a number and format it. :param x: a number :type x: int :param unit: an arbitrary unit name :type unit: str :return: scaled and formatted number :rtype: str """ # simple case for small numbers if x < 1000: return '%d %s' % (x, unit) # pick a scale (scale, suffix) = _pick_scale_and_suffix(x) # decide how many digits after the decimal to display scaled = x / scale if scaled < 10.0: fmt = '%1.2f %s%s' elif scaled < 100.0: fmt = '%1.1f %s%s' else: fmt = '%1.0f %s%s' # format it return fmt % (scaled, suffix, unit) def format_and_scale_fraction(numerator, denominator, unit): """ Pick a good scale for representing a fraction, and format it. :param numerator: a numerator of a fraction :type numerator: int :param denominator: a denominator of a fraction :type denominator: int :param unit: an arbitrary unit name :type unit: str :return: scaled and formatted fraction :rtype: str """ # simple case for small numbers if denominator < 1000: return '%d / %d %s' % (numerator, denominator, unit) # pick a scale (scale, suffix) = _pick_scale_and_suffix(denominator) # decide how many digits after the decimal to display scaled_denominator = denominator / scale if scaled_denominator < 10.0: fmt = '%1.2f / %1.2f %s%s' elif scaled_denominator < 100.0: fmt = '%1.1f / %1.1f %s%s' else: fmt = '%1.0f / %1.0f %s%s' # format it scaled_numerator = numerator / scale return fmt % (scaled_numerator, scaled_denominator, suffix, unit) _CAMELCASE_TO_UNDERSCORE_RE = re.compile('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))') def camelcase_to_underscore(input_): """ Convert a camel-cased string to a string with underscores. :param input_: an input string :type input_: str :return: string with underscores :rtype: str """ return _CAMELCASE_TO_UNDERSCORE_RE.sub(r'_\1', input_).lower() class B2TraceMeta(DefaultTraceMeta): """ Trace all public method calls, except for ones with names that begin with `get_`. """ pass class B2TraceMetaAbstract(DefaultTraceAbstractMeta): """ Default class for tracers, to be set as a metaclass for abstract base classes. """ pass class ConcurrentUsedAuthTokenGuard: """ Context manager preventing two tokens being used simultaneously. Throws UploadTokenUsedConcurrently when unable to acquire a lock Sample usage: with ConcurrentUsedAuthTokenGuard(lock_for_token, token): # code that uses the token exclusively """ def __init__(self, lock, token): self.lock = lock self.token = token def __enter__(self): if not self.lock.acquire(False): from b2sdk.exception import UploadTokenUsedConcurrently raise UploadTokenUsedConcurrently(self.token) def __exit__(self, exc_type, exc_val, exc_tb): try: self.lock.release() except RuntimeError: # guard against releasing a non-acquired lock pass def current_time_millis(): """ File times are in integer milliseconds, to avoid roundoff errors. """ return int(round(time.time() * 1000)) assert disable_trace assert limit_trace_arguments assert trace_call b2-sdk-python-1.17.3/b2sdk/utils/range_.py000066400000000000000000000031171426424117700201700ustar00rootroot00000000000000###################################################################### # # File: b2sdk/utils/range_.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### class Range: """ HTTP ranges use an *inclusive* index at the end. """ def __init__(self, start, end): assert 0 <= start <= end self.start = start self.end = end @classmethod def from_header(cls, raw_range_header): """ Factory method which returns an object constructed from Range http header. raw_range_header example: 'bytes=0-11' """ offsets = tuple( int(i) for i in raw_range_header.replace('bytes', '').strip('= ').split('-') ) return cls(*offsets) def size(self): return self.end - self.start + 1 def subrange(self, sub_start, sub_end): """ Return a range that is part of this range. :param sub_start: index relative to the start of this range. :param sub_end: (Inclusive!) index relative to the start of this range. :return: a new Range """ assert 0 <= sub_start <= sub_end < self.size() return self.__class__(self.start + sub_start, self.start + sub_end) def as_tuple(self): return self.start, self.end def __repr__(self): return '%s(%d, %d)' % (self.__class__.__name__, self.start, self.end) def __eq__(self, other): return self.start == other.start and self.end == other.end b2-sdk-python-1.17.3/b2sdk/utils/thread_pool.py000066400000000000000000000021171426424117700212340ustar00rootroot00000000000000###################################################################### # # File: b2sdk/utils/thread_pool.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from concurrent.futures import ThreadPoolExecutor from typing import Optional from b2sdk.utils import B2TraceMetaAbstract class ThreadPoolMixin(metaclass=B2TraceMetaAbstract): """ Mixin class with ThreadPoolExecutor. """ DEFAULT_THREAD_POOL_CLASS = staticmethod(ThreadPoolExecutor) def __init__( self, thread_pool: Optional[ThreadPoolExecutor] = None, max_workers: Optional[int] = None, **kwargs ): """ :param thread_pool: thread pool to be used :param max_workers: maximum number of worker threads (ignored if thread_pool is not None) """ self._thread_pool = thread_pool if thread_pool is not None \ else self.DEFAULT_THREAD_POOL_CLASS(max_workers=max_workers) super().__init__(**kwargs) b2-sdk-python-1.17.3/b2sdk/v0/000077500000000000000000000000001426424117700155465ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/v0/__init__.py000066400000000000000000000012361426424117700176610ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.v1 import * # noqa from b2sdk.v0.account_info import AbstractAccountInfo, InMemoryAccountInfo, UrlPoolAccountInfo, SqliteAccountInfo from b2sdk.v0.api import B2Api from b2sdk.v0.bucket import Bucket from b2sdk.v0.bucket import BucketFactory from b2sdk.v0.sync import Synchronizer from b2sdk.v0.sync import make_folder_sync_actions from b2sdk.v0.sync import sync_folders b2-sdk-python-1.17.3/b2sdk/v0/account_info.py000066400000000000000000000037411426424117700205740ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/account_info.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk import version_utils from b2sdk import v1 class OldAccountInfoMethods: """ this class contains proxy methods for deprecated signatures renamed for consistency in mid-2019 """ def get_account_id_or_app_key_id(self): """ Return the application key ID used to authenticate. :rtype: str .. deprecated:: 0.1.6 Use :func:`get_application_key_id` instead. """ return self.get_application_key_id() @version_utils.rename_argument( 'account_id_or_app_key_id', 'application_key_id', '0.1.5', None, ) def set_auth_data( self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, allowed=None, application_key_id=None, s3_api_url=None, ): # we need to enumerate all the parameters and cannot just "*args, **kwargs" because # the deprecation decorator doesn't feel safe with the kwargs approach return super(OldAccountInfoMethods, self).set_auth_data( account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, allowed, application_key_id, s3_api_url=s3_api_url, ) class AbstractAccountInfo(OldAccountInfoMethods, v1.AbstractAccountInfo): pass class InMemoryAccountInfo(OldAccountInfoMethods, v1.InMemoryAccountInfo): pass class UrlPoolAccountInfo(OldAccountInfoMethods, v1.UrlPoolAccountInfo): pass class SqliteAccountInfo(OldAccountInfoMethods, v1.SqliteAccountInfo): pass b2-sdk-python-1.17.3/b2sdk/v0/api.py000066400000000000000000000017041426424117700166730ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/api.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from .bucket import Bucket, BucketFactory from b2sdk import v1 class B2Api(v1.B2Api): BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) BUCKET_CLASS = staticmethod(Bucket) def delete_bucket(self, bucket): """ Delete the chosen bucket. For legacy reasons it returns whatever server sends in response, but API user should not rely on the response: if it doesn't raise an exception, it means that the operation was a success. :param b2sdk.v1.Bucket bucket: a :term:`bucket` to delete """ account_id = self.account_info.get_account_id() return self.session.delete_bucket(account_id, bucket.id_) b2-sdk-python-1.17.3/b2sdk/v0/bucket.py000066400000000000000000000020231426424117700173720ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/bucket.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk import v1 class Bucket(v1.Bucket): def list_file_names(self, start_filename=None, max_entries=None, prefix=None): """ Legacy interface which just returns whatever remote API returns. """ return self.api.session.list_file_names(self.id_, start_filename, max_entries, prefix) def list_file_versions( self, start_filename=None, start_file_id=None, max_entries=None, prefix=None ): """ Legacy interface which just returns whatever remote API returns. """ return self.api.session.list_file_versions( self.id_, start_filename, start_file_id, max_entries, prefix ) class BucketFactory(v1.BucketFactory): BUCKET_CLASS = staticmethod(Bucket) b2-sdk-python-1.17.3/b2sdk/v0/exception.py000066400000000000000000000016421426424117700201210ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### B2Error = None # calm down, pyflakes from b2sdk.v1.exception import * # noqa v1DestFileNewer = DestFileNewer # override to retain old style __str__ class DestFileNewer(v1DestFileNewer): def __str__(self): return 'source file is older than destination: %s%s with a time of %s cannot be synced to %s%s with a time of %s, unless --skipNewer or --replaceNewer is provided' % ( self.source_prefix, self.source_file.name, self.source_file.latest_version().mod_time, self.dest_prefix, self.dest_file.name, self.dest_file.latest_version().mod_time, ) b2-sdk-python-1.17.3/b2sdk/v0/sync.py000066400000000000000000000161411426424117700170770ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/sync.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging from b2sdk.v1.exception import CommandError from b2sdk.v1.exception import DestFileNewer as DestFileNewerV1 from b2sdk.v1 import trace_call from .exception import DestFileNewer from b2sdk.v1.exception import InvalidArgument, IncompleteSync from b2sdk.v1 import NewerFileSyncMode, CompareVersionMode from b2sdk.v1 import KeepOrDeleteMode from b2sdk.v1 import DEFAULT_SCAN_MANAGER from b2sdk.v1 import SyncReport from b2sdk.v1 import Synchronizer as SynchronizerV1 from b2sdk.v1 import AbstractSyncEncryptionSettingsProvider, SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER logger = logging.getLogger(__name__) class Synchronizer(SynchronizerV1): """ This is wrapper for newer version and will return the v0 style exceptions """ def __init__(self, *args, **kwargs): try: super(Synchronizer, self).__init__(*args, **kwargs) except InvalidArgument as e: raise CommandError('--%s %s' % (e.parameter_name, e.message)) def _make_file_sync_actions(self, *args, **kwargs): try: for i in super(Synchronizer, self)._make_file_sync_actions(*args, **kwargs): yield i except DestFileNewerV1 as e: raise DestFileNewer(e.dest_file, e.source_file, e.dest_prefix, e.source_prefix) def sync_folders(self, *args, **kwargs): try: super(Synchronizer, self).sync_folders(*args, **kwargs) except InvalidArgument as e: raise CommandError('--%s %s' % (e.parameter_name, e.message)) except IncompleteSync as e: raise CommandError(str(e)) def get_synchronizer_from_args( args, max_workers, policies_manager=DEFAULT_SCAN_MANAGER, dry_run=False, allow_empty_source=False, ): if args.replaceNewer and args.skipNewer: raise CommandError('--skipNewer and --replaceNewer are incompatible') elif args.replaceNewer: newer_file_mode = NewerFileSyncMode.REPLACE elif args.skipNewer: newer_file_mode = NewerFileSyncMode.SKIP else: newer_file_mode = NewerFileSyncMode.RAISE_ERROR if args.delete and (args.keepDays is not None): raise CommandError('--delete and --keepDays are incompatible') if args.compareVersions == 'none': compare_version_mode = CompareVersionMode.NONE elif args.compareVersions == 'modTime': compare_version_mode = CompareVersionMode.MODTIME elif args.compareVersions == 'size': compare_version_mode = CompareVersionMode.SIZE elif args.compareVersions is None: compare_version_mode = CompareVersionMode.MODTIME else: raise CommandError('Invalid option for --compareVersions') compare_threshold = args.compareThreshold keep_days = None if args.delete: keep_days_or_delete = KeepOrDeleteMode.DELETE elif args.keepDays: keep_days_or_delete = KeepOrDeleteMode.KEEP_BEFORE_DELETE keep_days = args.keepDays else: keep_days_or_delete = KeepOrDeleteMode.NO_DELETE return Synchronizer( max_workers, policies_manager=policies_manager, dry_run=dry_run, allow_empty_source=allow_empty_source, newer_file_mode=newer_file_mode, keep_days_or_delete=keep_days_or_delete, compare_version_mode=compare_version_mode, compare_threshold=compare_threshold, keep_days=keep_days, ) def make_folder_sync_actions( source_folder, dest_folder, args, now_millis, reporter, policies_manager=DEFAULT_SCAN_MANAGER, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ This is deprecated. Use the new Synchronizer class. Yields a sequence of actions that will sync the destination folder to the source folder. :param source_folder: source folder object :type source_folder: b2sdk.scan.folder.AbstractFolder :param dest_folder: destination folder object :type dest_folder: b2sdk.scan.folder.AbstractFolder :param args: an object which holds command line arguments :param now_millis: current time in milliseconds :type now_millis: int :param reporter: reporter object :param policies_manager: policies manager object :param encryption_settings_provider: encryption settings provider :type encryption_settings_provider: AbstractSyncEncryptionSettingsProvider """ synchronizer = get_synchronizer_from_args( args, 1, policies_manager=policies_manager, dry_run=False, allow_empty_source=False, ) try: return synchronizer.make_folder_sync_actions( source_folder, dest_folder, now_millis, reporter, policies_manager=policies_manager, encryption_settings_provider=encryption_settings_provider ) except InvalidArgument as e: raise CommandError('--%s %s' % (e.parameter_name, e.message)) @trace_call(logger) def sync_folders( source_folder, dest_folder, args, now_millis, stdout, no_progress, max_workers, policies_manager=DEFAULT_SCAN_MANAGER, dry_run=False, allow_empty_source=False, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ This is deprecated. Use the new Synchronizer class. source is also in the destination. Deletes any file versions in the destination older than history_days. :param source_folder: source folder object :type source_folder: b2sdk.scan.folder.AbstractFolder :param dest_folder: destination folder object :type dest_folder: b2sdk.scan.folder.AbstractFolder :param args: an object which holds command line arguments :param now_millis: current time in milliseconds :type now_millis: int :param stdout: standard output file object :param no_progress: if True, do not show progress :type no_progress: bool :param max_workers: max number of workers :type max_workers: int :param policies_manager: policies manager object :param dry_run: :type dry_run: bool :param allow_empty_source: if True, do not check whether source folder is empty :type allow_empty_source: bool :param encryption_settings_provider: encryption settings provider :type encryption_settings_provider: AbstractSyncEncryptionSettingsProvider """ synchronizer = get_synchronizer_from_args( args, max_workers, policies_manager=policies_manager, dry_run=dry_run, allow_empty_source=allow_empty_source, ) with SyncReport(stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder, dest_folder, now_millis, reporter, encryption_settings_provider=encryption_settings_provider ) b2-sdk-python-1.17.3/b2sdk/v1/000077500000000000000000000000001426424117700155475ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/v1/__init__.py000066400000000000000000000024441426424117700176640ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.v2 import * # noqa from b2sdk.v1.account_info import ( AbstractAccountInfo, InMemoryAccountInfo, UrlPoolAccountInfo, SqliteAccountInfo, StubAccountInfo ) from b2sdk.v1.api import B2Api from b2sdk.v1.b2http import B2Http from b2sdk.v1.bucket import Bucket, BucketFactory from b2sdk.v1.cache import AbstractCache from b2sdk.v1.download_dest import ( AbstractDownloadDestination, DownloadDestLocalFile, PreSeekedDownloadDest, DownloadDestBytes, DownloadDestProgressWrapper ) from b2sdk.v1.exception import CommandError, DestFileNewer from b2sdk.v1.file_metadata import FileMetadata from b2sdk.v1.file_version import FileVersionInfo from b2sdk.v1.session import B2Session from b2sdk.v1.sync import ( ScanPoliciesManager, DEFAULT_SCAN_MANAGER, zip_folders, Synchronizer, AbstractFolder, LocalFolder, B2Folder, parse_sync_folder, SyncReport, File, B2File, FileVersion, AbstractSyncEncryptionSettingsProvider ) from b2sdk.v1.replication.monitoring import ReplicationMonitor B2RawApi = B2RawHTTPApi b2-sdk-python-1.17.3/b2sdk/v1/account_info.py000066400000000000000000000133741426424117700206000ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/account_info.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import abstractmethod import inspect import logging import os from typing import Optional from b2sdk import v2 from b2sdk.account_info.sqlite_account_info import DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE from b2sdk.utils import limit_trace_arguments logger = logging.getLogger(__name__) # Retain legacy get_minimum_part_size and facilitate for optional s3_api_url class OldAccountInfoMethods: REALM_URLS = v2.REALM_URLS @limit_trace_arguments( only=[ 'self', 'api_url', 'download_url', 'minimum_part_size', 'realm', 's3_api_url', ] ) def set_auth_data( self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, allowed=None, application_key_id=None, s3_api_url=None, ): if 's3_api_url' in inspect.getfullargspec(self._set_auth_data).args: s3_kwargs = dict(s3_api_url=s3_api_url) else: s3_kwargs = {} if allowed is None: allowed = self.DEFAULT_ALLOWED assert self.allowed_is_valid(allowed) self._set_auth_data( account_id=account_id, auth_token=auth_token, api_url=api_url, download_url=download_url, minimum_part_size=minimum_part_size, application_key=application_key, realm=realm, allowed=allowed, application_key_id=application_key_id, **s3_kwargs, ) # translate legacy "minimum_part_size" to new style "recommended_part_size" class MinimumPartSizeTranslator: def _set_auth_data( self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, s3_api_url=None, allowed=None, application_key_id=None ): if 's3_api_url' in inspect.getfullargspec(super()._set_auth_data).args: s3_kwargs = dict(s3_api_url=s3_api_url) else: s3_kwargs = {} return super()._set_auth_data( account_id=account_id, auth_token=auth_token, api_url=api_url, download_url=download_url, recommended_part_size=minimum_part_size, absolute_minimum_part_size=DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE, application_key=application_key, realm=realm, allowed=allowed, application_key_id=application_key_id, **s3_kwargs, ) def get_minimum_part_size(self): return self.get_recommended_part_size() class AbstractAccountInfo(OldAccountInfoMethods, v2.AbstractAccountInfo): def get_s3_api_url(self): """ Return s3_api_url or raises MissingAccountData exception. :rtype: str """ # Removed @abstractmethod decorators def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> Optional[str]: """ Look up the bucket name for the given bucket id. """ # Removed @abstractmethod decorator def get_recommended_part_size(self): """ Return the recommended number of bytes in a part of a large file. :return: number of bytes :rtype: int """ # Removed @abstractmethod decorator def get_absolute_minimum_part_size(self): """ Return the absolute minimum number of bytes in a part of a large file. :return: number of bytes :rtype: int """ # Removed @abstractmethod decorator @abstractmethod def get_minimum_part_size(self): """ Return the minimum number of bytes in a part of a large file. :return: number of bytes :rtype: int """ # This stays abstract in v1 @abstractmethod def _set_auth_data( self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id ): """ Actually store the auth data. Can assume that 'allowed' is present and valid. All of the information returned by ``b2_authorize_account`` is saved, because all of it is needed at some point. """ # Keep the old signature class InMemoryAccountInfo(MinimumPartSizeTranslator, OldAccountInfoMethods, v2.InMemoryAccountInfo): pass class UrlPoolAccountInfo(OldAccountInfoMethods, v2.UrlPoolAccountInfo): pass class SqliteAccountInfo(MinimumPartSizeTranslator, OldAccountInfoMethods, v2.SqliteAccountInfo): def __init__(self, file_name=None, last_upgrade_to_run=None): """ If ``file_name`` argument is empty or ``None``, path from ``B2_ACCOUNT_INFO`` environment variable is used. If that is not available, a default of ``~/.b2_account_info`` is used. :param str file_name: The sqlite file to use; overrides the default. :param int last_upgrade_to_run: For testing only, override the auto-update on the db. """ # use legacy env var resolution, XDG not supported file_name = file_name or os.environ.get( v2.B2_ACCOUNT_INFO_ENV_VAR, v2.B2_ACCOUNT_INFO_DEFAULT_FILE ) super().__init__(file_name=file_name, last_upgrade_to_run=last_upgrade_to_run) class StubAccountInfo(MinimumPartSizeTranslator, OldAccountInfoMethods, v2.StubAccountInfo): REALM_URLS = {'production': 'http://production.example.com'} b2-sdk-python-1.17.3/b2sdk/v1/api.py000066400000000000000000000177411426424117700167040ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Any, Dict, Optional, overload, Tuple, List from .download_dest import AbstractDownloadDestination from b2sdk import v2 from b2sdk.api import Services from .account_info import AbstractAccountInfo from .bucket import Bucket, BucketFactory, download_file_and_return_info_dict from .cache import AbstractCache from .file_version import FileVersionInfo, FileVersionInfoFactory, file_version_info_from_id_and_name from .session import B2Session # override to use legacy no-request method of creating a bucket from bucket_id and retain `check_bucket_restrictions` # public API method # and to use v1.Bucket # and to retain cancel_large_file return type # and to retain old style download_file_by_id signature (allowing for the new one as well) and exception # and to retain old style get_file_info return type # and to accept old-style raw_api argument # and to retain old style create_key, delete_key and list_keys interfaces and behaviour class B2Api(v2.B2Api): SESSION_CLASS = staticmethod(B2Session) BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) BUCKET_CLASS = staticmethod(Bucket) FILE_VERSION_FACTORY_CLASS = staticmethod(FileVersionInfoFactory) def __init__( self, account_info: Optional[AbstractAccountInfo] = None, cache: Optional[AbstractCache] = None, raw_api: v2.B2RawHTTPApi = None, max_upload_workers: int = 10, max_copy_workers: int = 10, api_config: Optional[v2.B2HttpApiConfig] = None, ): """ Initialize the API using the given account info. :param account_info: To learn more about Account Info objects, see here :class:`~b2sdk.v1.SqliteAccountInfo` :param cache: It is used by B2Api to cache the mapping between bucket name and bucket ids. default is :class:`~b2sdk.cache.DummyCache` :param max_upload_workers: a number of upload threads :param max_copy_workers: a number of copy threads :param raw_api: :param api_config: """ self.session = self.SESSION_CLASS( account_info=account_info, cache=cache, raw_api=raw_api, api_config=api_config, ) self.file_version_factory = self.FILE_VERSION_FACTORY_CLASS(self) self.download_version_factory = self.DOWNLOAD_VERSION_FACTORY_CLASS(self) self.services = Services( self, max_upload_workers=max_upload_workers, max_copy_workers=max_copy_workers, ) def get_file_info(self, file_id: str) -> Dict[str, Any]: """ Gets info about file version. :param str file_id: the id of the file. """ return self.session.get_file_info_by_id(file_id) def get_bucket_by_id(self, bucket_id): """ Return a bucket object with a given ID. Unlike ``get_bucket_by_name``, this method does not need to make any API calls. :param str bucket_id: a bucket ID :return: a Bucket object :rtype: b2sdk.v1.Bucket """ return self.BUCKET_CLASS(self, bucket_id) def check_bucket_restrictions(self, bucket_name): """ Check to see if the allowed field from authorize-account has a bucket restriction. If it does, checks if the bucket_name for a given api call matches that. If not, it raises a :py:exc:`b2sdk.v1.exception.RestrictedBucket` error. :param str bucket_name: a bucket name :raises b2sdk.v1.exception.RestrictedBucket: if the account is not allowed to use this bucket """ self.check_bucket_name_restrictions(bucket_name) def cancel_large_file(self, file_id: str) -> FileVersionInfo: file_id_and_name = super().cancel_large_file(file_id) return file_version_info_from_id_and_name(file_id_and_name, self) @overload def download_file_by_id( self, file_id: str, download_dest: AbstractDownloadDestination, progress_listener: Optional[v2.AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[v2.EncryptionSetting] = None, ) -> dict: ... @overload def download_file_by_id( self, file_id: str, progress_listener: Optional[v2.AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[v2.EncryptionSetting] = None, ) -> v2.DownloadedFile: ... def download_file_by_id( self, file_id: str, download_dest: Optional[AbstractDownloadDestination] = None, progress_listener: Optional[v2.AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[v2.EncryptionSetting] = None, ): """ Download a file with the given ID. :param file_id: a file ID :param download_dest: an instance of the one of the following classes: \ :class:`~b2sdk.v1.DownloadDestLocalFile`,\ :class:`~b2sdk.v1.DownloadDestBytes`,\ :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` :param progress_listener: an instance of the one of the following classes: \ :class:`~b2sdk.v1.PartProgressReporter`,\ :class:`~b2sdk.v1.TqdmProgressListener`,\ :class:`~b2sdk.v1.SimpleProgressListener`,\ :class:`~b2sdk.v1.DoNothingProgressListener`,\ :class:`~b2sdk.v1.ProgressListenerForTest`,\ :class:`~b2sdk.v1.SyncFileReporter`,\ or any sub class of :class:`~b2sdk.v1.AbstractProgressListener` :param range_: a list of two integers, the first one is a start\ position, and the second one is the end position in the file :param encryption: encryption settings (``None`` if unknown) """ downloaded_file = super().download_file_by_id( file_id=file_id, progress_listener=progress_listener, range_=range_, encryption=encryption, ) if download_dest is not None: try: return download_file_and_return_info_dict(downloaded_file, download_dest, range_) except ValueError as ex: if ex.args == ('no strategy suitable for download was found!',): raise AssertionError('no strategy suitable for download was found!') raise else: return downloaded_file def list_keys(self, start_application_key_id=None) -> dict: """ List application keys. Perform a single request and return at most ``self.DEFAULT_LIST_KEY_COUNT`` keys, as well as the value to supply to the next call as ``start_application_key_id``, if not all keys were retrieved. :param start_application_key_id: an :term:`application key ID` to start from or ``None`` to start from the beginning """ account_id = self.account_info.get_account_id() return self.session.list_keys( account_id, max_key_count=self.DEFAULT_LIST_KEY_COUNT, start_application_key_id=start_application_key_id ) def create_key( self, capabilities: List[str], key_name: str, valid_duration_seconds: Optional[int] = None, bucket_id: Optional[str] = None, name_prefix: Optional[str] = None, ): return super().create_key( capabilities=capabilities, key_name=key_name, valid_duration_seconds=valid_duration_seconds, bucket_id=bucket_id, name_prefix=name_prefix, ).as_dict() def delete_key(self, application_key_id): return super().delete_key_by_id(application_key_id).as_dict() b2-sdk-python-1.17.3/b2sdk/v1/b2http.py000066400000000000000000000035441426424117700173320ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/b2http.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import requests from b2sdk import v2 # Overridden to retain old-style __init__ signature class B2Http(v2.B2Http): """ A wrapper for the requests module. Provides the operations needed to access B2, and handles retrying when the returned status is 503 Service Unavailable, 429 Too Many Requests, etc. The operations supported are: - post_json_return_json - post_content_return_json - get_content The methods that return JSON either return a Python dict or raise a subclass of B2Error. They can be used like this: .. code-block:: python try: response_dict = b2_http.post_json_return_json(url, headers, params) ... except B2Error as e: ... """ # timeout for HTTP GET/POST requests TIMEOUT = 1200 # 20 minutes as server-side copy can take time def __init__(self, requests_module=None, install_clock_skew_hook=True, user_agent_append=None): """ Initialize with a reference to the requests module, which makes it easy to mock for testing. :param requests_module: a reference to requests module :param bool install_clock_skew_hook: if True, install a clock skew hook :param str user_agent_append: if provided, the string will be appended to the User-Agent """ super().__init__( v2.B2HttpApiConfig( http_session_factory=(requests_module or requests).Session, install_clock_skew_hook=install_clock_skew_hook, user_agent_append=user_agent_append ) ) b2-sdk-python-1.17.3/b2sdk/v1/bucket.py000066400000000000000000000306611426424117700174040ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/bucket.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from contextlib import suppress from typing import Optional, overload, Tuple from .download_dest import AbstractDownloadDestination from .file_metadata import FileMetadata from .file_version import FileVersionInfo, FileVersionInfoFactory, file_version_info_from_download_version from b2sdk import v2 from b2sdk.utils import validate_b2_file_name # Overridden to retain the obsolete copy_file and start_large_file methods # and to retain old style FILE_VERSION_FACTORY attribute # and to retain old style download_file_by_name signature # and to retain old style download_file_by_id signature (allowing for the new one as well) # and to retain old style get_file_info_by_name return type # and to to adjust to old style B2Api.get_file_info return type # and to retain old style update return type class Bucket(v2.Bucket): FILE_VERSION_FACTORY = staticmethod(FileVersionInfoFactory) def copy_file( self, file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_encryption: Optional[v2.EncryptionSetting] = None, source_encryption: Optional[v2.EncryptionSetting] = None, file_retention: Optional[v2.FileRetentionSetting] = None, legal_hold: Optional[v2.LegalHold] = None, ): """ Creates a new file in this bucket by (server-side) copying from an existing file. :param str file_id: file ID of existing file :param str new_file_name: file name of the new file :param tuple[int,int],None bytes_range: start and end offsets (**inclusive!**), default is the entire file :param b2sdk.v1.MetadataDirectiveMode,None metadata_directive: default is :py:attr:`b2sdk.v1.MetadataDirectiveMode.COPY` :param str,None content_type: content_type for the new file if metadata_directive is set to :py:attr:`b2sdk.v1.MetadataDirectiveMode.REPLACE`, default will copy the content_type of old file :param dict,None file_info: file_info for the new file if metadata_directive is set to :py:attr:`b2sdk.v1.MetadataDirectiveMode.REPLACE`, default will copy the file_info of old file :param b2sdk.v1.EncryptionSetting destination_encryption: encryption settings for the destination (``None`` if unknown) :param b2sdk.v1.EncryptionSetting source_encryption: encryption settings for the source (``None`` if unknown) :param b2sdk.v1.FileRetentionSetting file_retention: retention setting for the new file :param bool legal_hold: legalHold setting for the new file """ return self.api.session.copy_file( file_id, new_file_name, bytes_range, metadata_directive, content_type, file_info, self.id_, destination_server_side_encryption=destination_encryption, source_server_side_encryption=source_encryption, file_retention=file_retention, legal_hold=legal_hold, ) def start_large_file( self, file_name, content_type=None, file_info=None, file_retention: Optional[v2.FileRetentionSetting] = None, legal_hold: Optional[v2.LegalHold] = None, ): """ Start a large file transfer. :param str file_name: a file name :param str,None content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param dict,None file_info: a file info to store with the file or ``None`` to not store anything :param b2sdk.v1.FileRetentionSetting file_retention: retention setting for the new file :param bool legal_hold: legalHold setting for the new file """ validate_b2_file_name(file_name) return self.api.services.large_file.start_large_file( self.id_, file_name, content_type=content_type, file_info=file_info, file_retention=file_retention, legal_hold=legal_hold, ) def download_file_by_name( self, file_name: str, download_dest: AbstractDownloadDestination, progress_listener: Optional[v2.AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[v2.EncryptionSetting] = None, ): """ Download a file by name. .. seealso:: :ref:`Synchronizer `, a *high-performance* utility that synchronizes a local folder with a Bucket. :param str file_name: a file name :param download_dest: an instance of the one of the following classes: \ :class:`~b2sdk.v1.DownloadDestLocalFile`,\ :class:`~b2sdk.v1.DownloadDestBytes`,\ :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` :param progress_listener: a progress listener object to use, or ``None`` to not track progress :param range_: two integer values, start and end offsets :param encryption: encryption settings (``None`` if unknown) """ downloaded_file = super().download_file_by_name( file_name=file_name, progress_listener=progress_listener, range_=range_, encryption=encryption, ) try: return download_file_and_return_info_dict(downloaded_file, download_dest, range_) except ValueError as ex: if ex.args == ('no strategy suitable for download was found!',): raise AssertionError('no strategy suitable for download was found!') raise @overload def download_file_by_id( self, file_id: str, download_dest: AbstractDownloadDestination = None, progress_listener: Optional[v2.AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[v2.EncryptionSetting] = None, ) -> dict: ... @overload def download_file_by_id( self, file_id: str, progress_listener: Optional[v2.AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[v2.EncryptionSetting] = None, ) -> v2.DownloadedFile: ... def download_file_by_id( self, file_id: str, download_dest: Optional[AbstractDownloadDestination] = None, progress_listener: Optional[v2.AbstractProgressListener] = None, range_: Optional[Tuple[int, int]] = None, encryption: Optional[v2.EncryptionSetting] = None, ): """ Download a file by ID. .. note:: download_file_by_id actually belongs in :py:class:`b2sdk.v1.B2Api`, not in :py:class:`b2sdk.v1.Bucket`; we just provide a convenient redirect here :param file_id: a file ID :param download_dest: an instance of the one of the following classes: \ :class:`~b2sdk.v1.DownloadDestLocalFile`,\ :class:`~b2sdk.v1.DownloadDestBytes`,\ :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` :param progress_listener: a progress listener object to use, or ``None`` to not report progress :param range_: two integer values, start and end offsets :param encryption: encryption settings (``None`` if unknown) """ return self.api.download_file_by_id( file_id, download_dest, progress_listener, range_=range_, encryption=encryption, ) def get_file_info_by_name(self, file_name: str) -> FileVersionInfo: return file_version_info_from_download_version(super().get_file_info_by_name(file_name)) def get_file_info_by_id(self, file_id: str) -> FileVersionInfo: """ Gets a file version's by ID. :param str file_id: the id of the file. """ return self.api.file_version_factory.from_api_response(self.api.get_file_info(file_id)) def update( self, bucket_type: Optional[str] = None, bucket_info: Optional[dict] = None, cors_rules: Optional[dict] = None, lifecycle_rules: Optional[list] = None, if_revision_is: Optional[int] = None, default_server_side_encryption: Optional[v2.EncryptionSetting] = None, default_retention: Optional[v2.BucketRetentionSetting] = None, **kwargs ): """ Update various bucket parameters. :param bucket_type: a bucket type, e.g. ``allPrivate`` or ``allPublic`` :param bucket_info: an info to store with a bucket :param cors_rules: CORS rules to store with a bucket :param lifecycle_rules: lifecycle rules of the bucket :param if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is* :param default_server_side_encryption: default server side encryption settings (``None`` if unknown) :param default_retention: bucket default retention setting """ # allow common tests to execute without hitting attributeerror with suppress(KeyError): del kwargs['replication'] self.replication = None assert not kwargs # after we get rid of everything we don't support in this apiver, this should be empty account_id = self.api.account_info.get_account_id() return self.api.session.update_bucket( account_id, self.id_, bucket_type=bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, ) def ls( self, folder_to_list: str = '', show_versions: bool = False, recursive: bool = False, fetch_count: Optional[int] = 10000 ): """ Pretend that folders exist and yields the information about the files in a folder. B2 has a flat namespace for the files in a bucket, but there is a convention of using "/" as if there were folders. This method searches through the flat namespace to find the files and "folders" that live within a given folder. When the `recursive` flag is set, lists all of the files in the given folder, and all of its sub-folders. :param folder_to_list: the name of the folder to list; must not start with "/". Empty string means top-level folder :param show_versions: when ``True`` returns info about all versions of a file, when ``False``, just returns info about the most recent versions :param recursive: if ``True``, list folders recursively :param fetch_count: how many entries to return or ``None`` to use the default. Acceptable values: 1 - 10000 :rtype: generator[tuple[b2sdk.v1.FileVersionInfo, str]] :returns: generator of (file_version, folder_name) tuples .. note:: In case of `recursive=True`, folder_name is returned only for first file in the folder. """ return super().ls(folder_to_list, not show_versions, recursive, fetch_count) def download_file_and_return_info_dict( downloaded_file: v2.DownloadedFile, download_dest: AbstractDownloadDestination, range_: Optional[Tuple[int, int]] ): with download_dest.make_file_context( file_id=downloaded_file.download_version.id_, file_name=downloaded_file.download_version.file_name, content_length=downloaded_file.download_version.size, content_type=downloaded_file.download_version.content_type, content_sha1=downloaded_file.download_version.content_sha1, file_info=downloaded_file.download_version.file_info, mod_time_millis=downloaded_file.download_version.mod_time_millis, range_=range_, ) as file: downloaded_file.save(file) return FileMetadata.from_download_version(downloaded_file.download_version).as_info_dict() class BucketFactory(v2.BucketFactory): BUCKET_CLASS = staticmethod(Bucket) b2-sdk-python-1.17.3/b2sdk/v1/cache.py000066400000000000000000000010161426424117700171620ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/cache.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional from b2sdk import v2 class AbstractCache(v2.AbstractCache): def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> Optional[str]: return None # Removed @abstractmethod decorator b2-sdk-python-1.17.3/b2sdk/v1/download_dest.py000066400000000000000000000155671426424117700207650ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/download_dest.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import io import os from abc import abstractmethod from contextlib import contextmanager from b2sdk.stream.progress import WritingStreamWithProgress from ..utils import B2TraceMetaAbstract, limit_trace_arguments, set_file_mtime class AbstractDownloadDestination(metaclass=B2TraceMetaAbstract): """ Interface to a destination for a downloaded file. """ @abstractmethod @limit_trace_arguments(skip=[ 'content_sha1', ]) def make_file_context( self, file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_=None ): """ Return a context manager that yields a binary file-like object to use for writing the contents of the file. :param str file_id: the B2 file ID from the headers :param str file_name: the B2 file name from the headers :param str content_length: the content length :param str content_type: the content type from the headers :param str content_sha1: the content sha1 from the headers (or ``"none"`` for large files) :param dict file_info: the user file info from the headers :param int mod_time_millis: the desired file modification date in ms since 1970-01-01 :param None,tuple[int,int] range_: starting and ending offsets of the received file contents. Usually ``None``, which means that the whole file is downloaded. :return: None """ class DownloadDestLocalFile(AbstractDownloadDestination): """ Store a downloaded file into a local file and sets its modification time. """ MODE = 'wb+' def __init__(self, local_file_path): self.local_file_path = local_file_path def make_file_context( self, file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_=None ): self.file_id = file_id self.file_name = file_name self.content_length = content_length self.content_type = content_type self.content_sha1 = content_sha1 self.file_info = file_info self.range_ = range_ return self.write_to_local_file_context(mod_time_millis) @contextmanager def write_to_local_file_context(self, mod_time_millis): completed = False try: # Open the file and let the caller write it. with io.open(self.local_file_path, self.MODE) as f: yield f # After it's closed, set the mod time. # This is an ugly hack to make the tests work. I can't think # of any other cases where set_file_mtime might fail. if self.local_file_path != os.devnull: set_file_mtime(self.local_file_path, mod_time_millis) # Set the flag that means to leave the downloaded file on disk. completed = True finally: # This is a best-effort attempt to clean up files that # failed to download, so we don't leave partial files # sitting on disk. if not completed: os.unlink(self.local_file_path) class PreSeekedDownloadDest(DownloadDestLocalFile): """ Store a downloaded file into a local file and sets its modification time. Does not truncate the target file, seeks to a given offset just after opening a descriptor. """ MODE = 'rb+' def __init__(self, local_file_path, seek_target): self._seek_target = seek_target super(PreSeekedDownloadDest, self).__init__(local_file_path) @contextmanager def write_to_local_file_context(self, *args, **kwargs): with super(PreSeekedDownloadDest, self).write_to_local_file_context(*args, **kwargs) as f: f.seek(self._seek_target) yield f class DownloadDestBytes(AbstractDownloadDestination): """ Store a downloaded file into bytes in memory. """ def __init__(self): self.bytes_written = None def make_file_context( self, file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_=None ): self.file_id = file_id self.file_name = file_name self.content_length = content_length self.content_type = content_type self.content_sha1 = content_sha1 self.file_info = file_info self.mod_time_millis = mod_time_millis self.range_ = range_ return self.capture_bytes_context() @contextmanager def capture_bytes_context(self): """ Remember the bytes written in self.bytes_written. """ # Make a place to store the data written bytes_io = io.BytesIO() # Let the caller write it yield bytes_io # Capture the result. The BytesIO object won't let you grab # the data after it's closed self.bytes_written = bytes_io.getvalue() bytes_io.close() def get_bytes_written(self): if self.bytes_written is None: raise Exception('data not written yet') return self.bytes_written class DownloadDestProgressWrapper(AbstractDownloadDestination): """ Wrap a DownloadDestination and report progress to a ProgressListener. """ def __init__(self, download_dest, progress_listener): self.download_dest = download_dest self.progress_listener = progress_listener def make_file_context( self, file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_=None ): return self.write_file_and_report_progress_context( file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_ ) @contextmanager def write_file_and_report_progress_context( self, file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_ ): with self.download_dest.make_file_context( file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_ ) as file_: total_bytes = content_length if range_ is not None: total_bytes = range_[1] - range_[0] + 1 self.progress_listener.set_total_bytes(total_bytes) with self.progress_listener: yield WritingStreamWithProgress(file_, self.progress_listener) b2-sdk-python-1.17.3/b2sdk/v1/exception.py000066400000000000000000000030271426424117700201210ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.v2.exception import * # noqa v2DestFileNewer = DestFileNewer # This exception class is deprecated and should not be used in new designs class CommandError(B2Error): """ b2 command error (user caused). Accepts exactly one argument: message. We expect users of shell scripts will parse our ``__str__`` output. """ def __init__(self, message): super(CommandError, self).__init__() self.message = message def __str__(self): return self.message class DestFileNewer(v2DestFileNewer): def __init__(self, dest_file, source_file, dest_prefix, source_prefix): super(v2DestFileNewer, self).__init__() self.dest_file = dest_file self.source_file = source_file self.dest_prefix = dest_prefix self.source_prefix = source_prefix def __str__(self): return 'source file is older than destination: %s%s with a time of %s cannot be synced to %s%s with a time of %s, unless a valid newer_file_mode is provided' % ( self.source_prefix, self.source_file.name, self.source_file.latest_version().mod_time, self.dest_prefix, self.dest_file.name, self.dest_file.latest_version().mod_time, ) b2-sdk-python-1.17.3/b2sdk/v1/file_metadata.py000066400000000000000000000042151426424117700207020ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/file_metadata.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import b2sdk.v2 as v2 class FileMetadata: """ Hold information about a file which is being downloaded. """ UNVERIFIED_CHECKSUM_PREFIX = 'unverified:' def __init__( self, file_id, file_name, content_type, content_length, content_sha1, file_info, ): self.file_id = file_id self.file_name = file_name self.content_type = content_type self.content_length = content_length self.content_sha1, self.content_sha1_verified = self._decode_content_sha1(content_sha1) self.file_info = file_info def as_info_dict(self): return { 'fileId': self.file_id, 'fileName': self.file_name, 'contentType': self.content_type, 'contentLength': self.content_length, 'contentSha1': self._encode_content_sha1(self.content_sha1, self.content_sha1_verified), 'fileInfo': self.file_info, } @classmethod def _decode_content_sha1(cls, content_sha1): if content_sha1.startswith(cls.UNVERIFIED_CHECKSUM_PREFIX): return content_sha1[len(cls.UNVERIFIED_CHECKSUM_PREFIX):], False return content_sha1, True @classmethod def _encode_content_sha1(cls, content_sha1, content_sha1_verified): if not content_sha1_verified: return '%s%s' % (cls.UNVERIFIED_CHECKSUM_PREFIX, content_sha1) return content_sha1 @classmethod def from_download_version(cls, download_version: v2.DownloadVersion): return cls( file_id=download_version.id_, file_name=download_version.file_name, content_type=download_version.content_type, content_length=download_version.content_length, content_sha1=download_version.content_sha1, file_info=download_version.file_info, ) b2-sdk-python-1.17.3/b2sdk/v1/file_version.py000066400000000000000000000150521426424117700206100ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/file_version.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from contextlib import suppress from typing import Optional import datetime import functools from b2sdk import v2 from . import api as v1api # Override to retain legacy class name, __init__ signature, slots # and old formatting methods # and to omit 'api' property when doing __eq__ and __repr__ # and to make get_fresh_state return proper objects, even though v1.B2Api.get_file_info returns dicts class FileVersionInfo(v2.FileVersion): __slots__ = ['_api'] LS_ENTRY_TEMPLATE = '%83s %6s %10s %8s %9d %s' # order is file_id, action, date, time, size, name def __init__( self, id_, file_name, size, content_type, content_sha1, file_info, upload_timestamp, action, account_id: Optional[str] = None, bucket_id: Optional[str] = None, content_md5=None, server_side_encryption: Optional[v2.EncryptionSetting] = None, file_retention: Optional[v2.FileRetentionSetting] = None, legal_hold: Optional[v2.LegalHold] = None, api: Optional['v1api.B2Api'] = None, **kwargs ): self.id_ = id_ self.file_name = file_name self.size = size and int(size) self.content_type = content_type self.content_sha1, self.content_sha1_verified = self._decode_content_sha1(content_sha1) self.account_id = account_id self.bucket_id = bucket_id self.content_md5 = content_md5 self.file_info = file_info or {} self.upload_timestamp = upload_timestamp self.action = action self.server_side_encryption = server_side_encryption self.legal_hold = legal_hold self.file_retention = file_retention self._api = api # allow common tests to execute without hitting attributeerror with suppress(KeyError): del kwargs['replication_status'] self.replication_status = None assert not kwargs # after we get rid of everything we don't support in this apiver, this should be empty if v2.SRC_LAST_MODIFIED_MILLIS in self.file_info: self.mod_time_millis = int(self.file_info[v2.SRC_LAST_MODIFIED_MILLIS]) else: self.mod_time_millis = self.upload_timestamp @property def api(self): if self._api is None: raise ValueError('"api" not set') return self._api def _all_slots(self): all_slots = super()._all_slots() all_slots.remove('api') return all_slots def format_ls_entry(self): dt = datetime.datetime.utcfromtimestamp(self.upload_timestamp / 1000) date_str = dt.strftime('%Y-%m-%d') time_str = dt.strftime('%H:%M:%S') size = self.size or 0 # required if self.action == 'hide' return self.LS_ENTRY_TEMPLATE % ( self.id_, self.action, date_str, time_str, size, self.file_name, ) @classmethod def format_folder_ls_entry(cls, name): return cls.LS_ENTRY_TEMPLATE % ('-', '-', '-', '-', 0, name) def get_fresh_state(self) -> 'FileVersionInfo': """ Fetch all the information about this file version and return a new FileVersion object. This method does NOT change the object it is called on. """ return self.api.file_version_factory.from_api_response(self.api.get_file_info(self.id_)) def file_version_info_from_new_file_version(file_version: v2.FileVersion) -> FileVersionInfo: return FileVersionInfo( **{ att_name: getattr(file_version, att_name) for att_name in [ 'id_', 'file_name', 'size', 'content_type', 'content_sha1', 'file_info', 'upload_timestamp', 'action', 'content_md5', 'server_side_encryption', 'legal_hold', 'file_retention', 'api', ] } ) def translate_single_file_version(func): @functools.wraps(func) def inner(*a, **kw): return file_version_info_from_new_file_version(func(*a, **kw)) return inner # override to return old style FileVersionInfo class FileVersionInfoFactory(v2.FileVersionFactory): from_api_response = translate_single_file_version(v2.FileVersionFactory.from_api_response) def from_response_headers(self, headers): file_info = v2.DownloadVersionFactory.file_info_from_headers(headers) return FileVersionInfo( api=self.api, id_=headers['x-bz-file-id'], file_name=headers['x-bz-file-name'], size=int(headers['content-length']), content_type=headers['content-type'], content_sha1=headers['x-bz-content-sha1'], file_info=file_info, upload_timestamp=int(headers['x-bz-upload-timestamp']), action='upload', content_md5=None, server_side_encryption=v2.EncryptionSettingFactory.from_response_headers(headers), file_retention=v2.FileRetentionSetting.from_response_headers(headers), legal_hold=v2.LegalHold.from_response_headers(headers), ) def file_version_info_from_id_and_name(file_id_and_name: v2.FileIdAndName, api: 'v1api.B2Api'): return FileVersionInfo( id_=file_id_and_name.file_id, file_name=file_id_and_name.file_name, size=0, content_type='unknown', content_sha1='none', file_info={}, upload_timestamp=0, action='cancel', api=api, ) def file_version_info_from_download_version(download_version: v2.DownloadVersion): return FileVersionInfo( id_=download_version.id_, file_name=download_version.file_name, size=download_version.size, content_type=download_version.content_type, content_sha1=download_version.content_sha1, file_info=download_version.file_info, upload_timestamp=download_version.upload_timestamp, action='upload', content_md5=None, server_side_encryption=download_version.server_side_encryption, file_retention=download_version.file_retention, legal_hold=download_version.legal_hold, api=download_version.api, ) b2-sdk-python-1.17.3/b2sdk/v1/replication/000077500000000000000000000000001426424117700200605ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/v1/replication/__init__.py000066400000000000000000000004531426424117700221730ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/replication/__init__.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/b2sdk/v1/replication/monitoring.py000066400000000000000000000021311426424117700226140ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/replication/monitoring.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from dataclasses import dataclass import b2sdk.v2 as v2 from .. import Bucket from ..sync.folder import B2Folder @dataclass class ReplicationMonitor(v2.ReplicationMonitor): # when passing in v1 Bucket objects to ReplicationMonitor, # the latter should use v1 B2Folder to correctly use # v1 Bucket's interface B2_FOLDER_CLASS = B2Folder @property def destination_bucket(self) -> Bucket: destination_api = self.destination_api or self.source_api bucket_id = self.rule.destination_bucket_id # when using `destination_api.get_bucket_by_id(bucket_id)`, # v1 will instantiate the bucket without its name, but we need it, # so we use `list_buckets` to actually fetch bucket name return destination_api.list_buckets(bucket_id=bucket_id)[0] b2-sdk-python-1.17.3/b2sdk/v1/session.py000066400000000000000000000050411426424117700176040ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/session.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional from b2sdk import v2 from b2sdk.v2.exception import InvalidArgument from .account_info import SqliteAccountInfo # Override to use legacy signature of account_info.set_auth_data, especially the minimum_part_size argument # and to accept old-style raw_api argument class B2Session(v2.B2Session): SQLITE_ACCOUNT_INFO_CLASS = staticmethod(SqliteAccountInfo) def __init__( self, account_info=None, cache=None, raw_api: v2.B2RawHTTPApi = None, api_config: Optional[v2.B2HttpApiConfig] = None ): if raw_api is not None and api_config is not None: raise InvalidArgument( 'raw_api,api_config', 'Provide at most one of: raw_api, api_config' ) if api_config is None: api_config = v2.DEFAULT_HTTP_API_CONFIG super().__init__(account_info=account_info, cache=cache, api_config=api_config) if raw_api is not None: self.raw_api = raw_api def authorize_account(self, realm, application_key_id, application_key): """ Perform account authorization. :param str realm: a realm to authorize account in (usually just "production") :param str application_key_id: :term:`application key ID` :param str application_key: user's :term:`application key` """ # Authorize realm_url = self.account_info.REALM_URLS.get(realm, realm) response = self.raw_api.authorize_account(realm_url, application_key_id, application_key) account_id = response['accountId'] allowed = response['allowed'] # Clear the cache if new account has been used if not self.account_info.is_same_account(account_id, realm): self.cache.clear() # Store the auth data self.account_info.set_auth_data( account_id=account_id, auth_token=response['authorizationToken'], api_url=response['apiUrl'], download_url=response['downloadUrl'], minimum_part_size=response['recommendedPartSize'], application_key=application_key, realm=realm, s3_api_url=response['s3ApiUrl'], allowed=allowed, application_key_id=application_key_id ) b2-sdk-python-1.17.3/b2sdk/v1/sync/000077500000000000000000000000001426424117700165235ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/v1/sync/__init__.py000066400000000000000000000007261426424117700206410ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from .encryption_provider import * from .file import * from .folder import * from .folder_parser import * from .report import * from .scan_policies import * from .sync import * b2-sdk-python-1.17.3/b2sdk/v1/sync/encryption_provider.py000066400000000000000000000075431426424117700232120ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/encryption_provider.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import inspect from abc import abstractmethod from typing import Optional from b2sdk import v2 from ..bucket import Bucket from ..file_version import FileVersionInfo # wrapper to translate new argument names to old ones class SyncEncryptionSettingsProviderWrapper(v2.AbstractSyncEncryptionSettingsProvider): def __init__(self, provider): self.provider = provider def __repr__(self): return "%s(%s)" % ( self.__class__.__name__, self.provider, ) def get_setting_for_upload( self, bucket: Bucket, b2_file_name: str, file_info: Optional[dict], length: int, ) -> Optional[v2.EncryptionSetting]: return self.provider.get_setting_for_upload( bucket=bucket, b2_file_name=b2_file_name, file_info=file_info, length=length, ) def get_source_setting_for_copy( self, bucket: Bucket, source_file_version: v2.FileVersion, ) -> Optional[v2.EncryptionSetting]: return self.provider.get_source_setting_for_copy( bucket=bucket, source_file_version_info=source_file_version ) def get_destination_setting_for_copy( self, bucket: Bucket, dest_b2_file_name: str, source_file_version: v2.FileVersion, target_file_info: Optional[dict] = None, ) -> Optional[v2.EncryptionSetting]: return self.provider.get_destination_setting_for_copy( bucket=bucket, dest_b2_file_name=dest_b2_file_name, source_file_version_info=source_file_version, target_file_info=target_file_info, ) def get_setting_for_download( self, bucket: Bucket, file_version: v2.FileVersion, ) -> Optional[v2.EncryptionSetting]: return self.provider.get_setting_for_download( bucket=bucket, file_version_info=file_version, ) def wrap_if_necessary(provider): if 'file_version' in inspect.getfullargspec(provider.get_setting_for_download).args: return provider return SyncEncryptionSettingsProviderWrapper(provider) # Old signatures class AbstractSyncEncryptionSettingsProvider(v2.AbstractSyncEncryptionSettingsProvider): @abstractmethod def get_setting_for_upload( self, bucket: Bucket, b2_file_name: str, file_info: Optional[dict], length: int, ) -> Optional[v2.EncryptionSetting]: """ Return an EncryptionSetting for uploading an object or None if server should decide. """ @abstractmethod def get_source_setting_for_copy( self, bucket: Bucket, source_file_version_info: FileVersionInfo, ) -> Optional[v2.EncryptionSetting]: """ Return an EncryptionSetting for a source of copying an object or None if not required """ @abstractmethod def get_destination_setting_for_copy( self, bucket: Bucket, dest_b2_file_name: str, source_file_version_info: FileVersionInfo, target_file_info: Optional[dict] = None, ) -> Optional[v2.EncryptionSetting]: """ Return an EncryptionSetting for a destination for copying an object or None if server should decide """ @abstractmethod def get_setting_for_download( self, bucket: Bucket, file_version_info: FileVersionInfo, ) -> Optional[v2.EncryptionSetting]: """ Return an EncryptionSetting for downloading an object from, or None if not required """ b2-sdk-python-1.17.3/b2sdk/v1/sync/file.py000066400000000000000000000073651426424117700200270ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/file.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import List from b2sdk.v1 import FileVersionInfo from b2sdk.http_constants import SRC_LAST_MODIFIED_MILLIS # This whole module is here to retain legacy classes so they can be used in retained legacy exception class File: """ Hold information about one file in a folder. The name is relative to the folder in all cases. Files that have multiple versions (which only happens in B2, not in local folders) include information about all of the versions, most recent first. """ __slots__ = ['name', 'versions'] def __init__(self, name, versions: List['FileVersion']): """ :param str name: a relative file name :param List[FileVersion] versions: a list of file versions """ self.name = name self.versions = versions def latest_version(self) -> 'FileVersion': """ Return the latest file version. """ return self.versions[0] def __repr__(self): return '%s(%s, [%s])' % ( self.__class__.__name__, self.name, ', '.join(repr(v) for v in self.versions) ) class B2File(File): """ Hold information about one file in a folder in B2 cloud. """ __slots__ = ['name', 'versions'] def __init__(self, name, versions: List['FileVersion']): """ :param str name: a relative file name :param List[FileVersion] versions: a list of file versions """ super().__init__(name, versions) def latest_version(self) -> 'FileVersion': return super().latest_version() class FileVersion: """ Hold information about one version of a file. """ __slots__ = ['id_', 'name', 'mod_time', 'action', 'size'] def __init__(self, id_, file_name, mod_time, action, size): """ :param id_: the B2 file id, or the local full path name :type id_: str :param file_name: a relative file name :type file_name: str :param mod_time: modification time, in milliseconds, to avoid rounding issues with millisecond times from B2 :type mod_time: int :param action: "hide" or "upload" (never "start") :type action: str :param size: a file size :type size: int """ self.id_ = id_ self.name = file_name self.mod_time = mod_time self.action = action self.size = size def __repr__(self): return '%s(%s, %s, %s, %s)' % ( self.__class__.__name__, repr(self.id_), repr(self.name), repr(self.mod_time), repr(self.action), ) class B2FileVersion(FileVersion): __slots__ = [ 'file_version_info' ] # in a typical use case there is a lot of these object in memory, hence __slots__ # and properties def __init__(self, file_version_info: FileVersionInfo): self.file_version_info = file_version_info @property def id_(self): return self.file_version_info.id_ @property def name(self): return self.file_version_info.file_name @property def mod_time(self): if SRC_LAST_MODIFIED_MILLIS in self.file_version_info.file_info: return int(self.file_version_info.file_info[SRC_LAST_MODIFIED_MILLIS]) return self.file_version_info.upload_timestamp @property def action(self): return self.file_version_info.action @property def size(self): return self.file_version_info.size b2-sdk-python-1.17.3/b2sdk/v1/sync/file_to_path_translator.py000066400000000000000000000053671426424117700240160ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/file_to_path_translator.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Tuple from b2sdk import v2 from .file import B2File, B2FileVersion, File, FileVersion # The goal is to create v1.File objects together with v1.FileVersion objects from v2.SyncPath objects def make_files_from_paths( dest_path: v2.AbstractSyncPath, source_path: v2.AbstractSyncPath, sync_type: str ) -> Tuple[File, File]: assert sync_type in ('b2-to-b2', 'b2-to-local', 'local-to-b2') sync_type_split = sync_type.split('-') dest_type = sync_type_split[-1] dest_file = _path_translation_map[dest_type](dest_path) source_type = sync_type_split[0] source_file = _path_translation_map[source_type](source_path) return dest_file, source_file def _translate_b2_path_to_file(path: v2.B2SyncPath) -> B2File: versions = [B2FileVersion(version) for version in path.all_versions] return B2File(path.relative_path, versions) def _translate_local_path_to_file(path: v2.LocalSyncPath) -> File: version = FileVersion( id_=path.absolute_path, file_name=path.relative_path, mod_time=path.mod_time, action='upload', size=path.size, ) return File(path.relative_path, [version]) _path_translation_map = {'b2': _translate_b2_path_to_file, 'local': _translate_local_path_to_file} # The goal is to create v2.SyncPath objects from v1.File objects def make_paths_from_files(dest_file: File, source_file: File, sync_type: str) -> Tuple[v2.AbstractSyncPath, v2.AbstractSyncPath]: assert sync_type in ('b2-to-b2', 'b2-to-local', 'local-to-b2') sync_type_split = sync_type.split('-') dest_type = sync_type_split[-1] dest_path = _file_translation_map[dest_type](dest_file) source_type = sync_type_split[0] source_path = _file_translation_map[source_type](source_file) return dest_path, source_path def _translate_b2_file_to_path(file: B2File) -> v2.AbstractSyncPath: versions = [file_version.file_version_info for file_version in file.versions] return v2.B2SyncPath( relative_path=file.name, selected_version=versions[0], all_versions=versions ) def _translate_local_file_to_path(file: File) -> v2.AbstractSyncPath: return v2.LocalSyncPath( absolute_path=file.latest_version().id_, relative_path=file.name, mod_time=file.latest_version().mod_time, size=file.latest_version().size ) _file_translation_map = {'b2': _translate_b2_file_to_path, 'local': _translate_local_file_to_path} b2-sdk-python-1.17.3/b2sdk/v1/sync/folder.py000066400000000000000000000043721426424117700203560ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/folder.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import abstractmethod import functools from b2sdk import v2 from .scan_policies import DEFAULT_SCAN_MANAGER, wrap_if_necessary from .. import exception def translate_errors(func): @functools.wraps(func) def wrapper(*a, **kw): try: return func(*a, **kw) except exception.NotADirectory as ex: raise Exception('%s is not a directory' % (ex.path,)) except exception.UnableToCreateDirectory as ex: raise Exception('unable to create directory %s' % (ex.path,)) except exception.EmptyDirectory as ex: raise exception.CommandError( 'Directory %s is empty. Use --allowEmptySource to sync anyway.' % (ex.path,) ) return wrapper # Override to change "policies_manager" default argument class AbstractFolder(v2.AbstractFolder): @abstractmethod def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): pass # override to retain "policies_manager" default argument, # and wrap policies_manager class B2Folder(v2.B2Folder, AbstractFolder): def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): return super().all_files(reporter, wrap_if_necessary(policies_manager)) def get_file_versions(self): for file_version, _ in self.bucket.ls( self.folder_name, show_versions=True, recursive=True, ): yield file_version # override to retain "policies_manager" default argument, # translate nice errors to old style Exceptions and CommandError # and wrap policies_manager class LocalFolder(v2.LocalFolder, AbstractFolder): @translate_errors def ensure_present(self): return super().ensure_present() @translate_errors def ensure_non_empty(self): return super().ensure_non_empty() def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): return super().all_files(reporter, wrap_if_necessary(policies_manager)) b2-sdk-python-1.17.3/b2sdk/v1/sync/folder_parser.py000066400000000000000000000012771426424117700217330ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/folder_parser.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk import v2 from .. import exception from .folder import LocalFolder, B2Folder # Override to use v1 version of "LocalFolder" and "B2Folder" and raise old style CommandError def parse_sync_folder(folder_name, api): try: return v2.parse_sync_folder(folder_name, api, LocalFolder, B2Folder) except exception.InvalidArgument as ex: raise exception.CommandError(ex.message) b2-sdk-python-1.17.3/b2sdk/v1/sync/report.py000066400000000000000000000014531426424117700204130ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/report.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk import v2 # override to retain legacy methods class SyncReport(v2.SyncReport): @property def local_file_count(self): return self.total_count @local_file_count.setter def local_file_count(self, value): self.total_count = value @property def local_done(self): return self.total_done @local_done.setter def local_done(self, value): self.total_done = value update_local = v2.SyncReport.update_total end_local = v2.SyncReport.end_total b2-sdk-python-1.17.3/b2sdk/v1/sync/scan_policies.py000066400000000000000000000155061426424117700217170ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/scan_policies.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import re from typing import Optional, Union, Iterable from .file import B2FileVersion from ..file_version import FileVersionInfo from .file_to_path_translator import _translate_local_path_to_file from b2sdk import v2 from b2sdk.v2 import exception as v2_exception # noqa # Override to retain old exceptions in __init__ # and to provide interface for new should_exclude_* methods class ScanPoliciesManager(v2.ScanPoliciesManager): """ Policy object used when scanning folders for syncing, used to decide which files to include in the list of files to be synced. Code that scans through files should at least use should_exclude_file() to decide whether each file should be included; it will check include/exclude patterns for file names, as well as patterns for excluding directories. Code that scans may optionally use should_exclude_directory() to test whether it can skip a directory completely and not bother listing the files and sub-directories in it. """ def __init__( self, exclude_dir_regexes: Iterable[Union[str, re.Pattern]] = tuple(), exclude_file_regexes: Iterable[Union[str, re.Pattern]] = tuple(), include_file_regexes: Iterable[Union[str, re.Pattern]] = tuple(), exclude_all_symlinks: bool = False, exclude_modified_before: Optional[int] = None, exclude_modified_after: Optional[int] = None, exclude_uploaded_before: Optional[int] = None, exclude_uploaded_after: Optional[int] = None, ): """ :param exclude_dir_regexes: regexes to exclude directories :param exclude_file_regexes: regexes to exclude files :param include_file_regexes: regexes to include files :param exclude_all_symlinks: if True, exclude all symlinks :param exclude_modified_before: optionally exclude file versions (both local and b2) modified before (in millis) :param exclude_modified_after: optionally exclude file versions (both local and b2) modified after (in millis) :param exclude_uploaded_before: optionally exclude b2 file versions uploaded before (in millis) :param exclude_uploaded_after: optionally exclude b2 file versions uploaded after (in millis) The regex matching priority for a given path is: 1) the path is always excluded if it's dir matches `exclude_dir_regexes`, if not then 2) the path is always included if it matches `include_file_regexes`, if not then 3) the path is excluded if it matches `exclude_file_regexes`, if not then 4) the path is included """ if include_file_regexes and not exclude_file_regexes: raise v2_exception.InvalidArgument( 'include_file_regexes', 'cannot be used without exclude_file_regexes at the same time' ) self._exclude_dir_set = v2.RegexSet(exclude_dir_regexes) self._exclude_file_because_of_dir_set = v2.RegexSet( map(v2.convert_dir_regex_to_dir_prefix_regex, exclude_dir_regexes) ) self._exclude_file_set = v2.RegexSet(exclude_file_regexes) self._include_file_set = v2.RegexSet(include_file_regexes) self.exclude_all_symlinks = exclude_all_symlinks self._include_mod_time_range = v2.IntegerRange( exclude_modified_before, exclude_modified_after ) with v2_exception.check_invalid_argument( 'exclude_uploaded_before,exclude_uploaded_after', '', ValueError ): self._include_upload_time_range = v2.IntegerRange( exclude_uploaded_before, exclude_uploaded_after ) def should_exclude_file(self, file_path): """ Given the full path of a file, decide if it should be excluded from the scan. :param file_path: the path of the file, relative to the root directory being scanned. :type: str :return: True if excluded. :rtype: bool """ if self._exclude_file_because_of_dir_set.matches(file_path): return True if self._include_file_set.matches(file_path): return False return self._exclude_file_set.matches(file_path) def should_exclude_file_version(self, file_version): """ Given the modification time of a file version, decide if it should be excluded from the scan. :param file_version: the file version object :type: b2sdk.v1.FileVersion :return: True if excluded. :rtype: bool """ return file_version.mod_time not in self._include_mod_time_range def should_exclude_directory(self, dir_path): """ Given the full path of a directory, decide if all of the files in it should be excluded from the scan. :param dir_path: the path of the directory, relative to the root directory being scanned. The path will never end in '/'. :type dir_path: str :return: True if excluded. """ return self._exclude_dir_set.matches(dir_path) class ScanPoliciesManagerWrapper(v2.ScanPoliciesManager): def __init__(self, scan_policies_manager: ScanPoliciesManager): self.scan_policies_manager = scan_policies_manager self.exclude_all_symlinks = scan_policies_manager.exclude_all_symlinks def __repr__(self): return "%s(%s)" % ( self.__class__.__name__, self.scan_policies_manager, ) def should_exclude_local_path(self, local_path: v2.LocalSyncPath): if self.scan_policies_manager.should_exclude_file_version( _translate_local_path_to_file(local_path).latest_version() ): return True return self.scan_policies_manager.should_exclude_file(local_path.relative_path) def should_exclude_b2_file_version(self, file_version: FileVersionInfo, relative_path: str): if self.scan_policies_manager.should_exclude_file_version(B2FileVersion(file_version)): return True return self.scan_policies_manager.should_exclude_file(relative_path) def should_exclude_b2_directory(self, dir_path): return self.scan_policies_manager.should_exclude_directory(dir_path) def should_exclude_local_directory(self, dir_path): return self.scan_policies_manager.should_exclude_directory(dir_path) def wrap_if_necessary(scan_policies_manager): if hasattr(scan_policies_manager, 'should_exclude_file'): return ScanPoliciesManagerWrapper(scan_policies_manager) return scan_policies_manager DEFAULT_SCAN_MANAGER = ScanPoliciesManager() b2-sdk-python-1.17.3/b2sdk/v1/sync/sync.py000066400000000000000000000123501426424117700200520ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/sync.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk import v2 from b2sdk.v2 import exception as v2_exception from .file_to_path_translator import make_files_from_paths, make_paths_from_files from .scan_policies import DEFAULT_SCAN_MANAGER, wrap_if_necessary as scan_wrap_if_necessary from .encryption_provider import wrap_if_necessary as encryption_wrap_if_necessary from ..exception import DestFileNewer # Override to change "policies_manager" default argument def zip_folders(folder_a, folder_b, reporter, policies_manager=DEFAULT_SCAN_MANAGER): return v2.zip_folders( folder_a, folder_b, reporter, policies_manager=scan_wrap_if_necessary(policies_manager) ) # Override to change "policies_manager" default arguments # and to wrap encryption_settings_providers in argument name translators class Synchronizer(v2.Synchronizer): def __init__( self, max_workers, policies_manager=DEFAULT_SCAN_MANAGER, dry_run=False, allow_empty_source=False, newer_file_mode=v2.NewerFileSyncMode.RAISE_ERROR, keep_days_or_delete=v2.KeepOrDeleteMode.NO_DELETE, compare_version_mode=v2.CompareVersionMode.MODTIME, compare_threshold=None, keep_days=None, sync_policy_manager: v2.SyncPolicyManager = v2.POLICY_MANAGER, ): super().__init__( max_workers, scan_wrap_if_necessary(policies_manager), dry_run, allow_empty_source, newer_file_mode, keep_days_or_delete, compare_version_mode, compare_threshold, keep_days, sync_policy_manager, ) def make_folder_sync_actions( self, source_folder, dest_folder, now_millis, reporter, policies_manager=DEFAULT_SCAN_MANAGER, encryption_settings_provider=v2.SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): return super()._make_folder_sync_actions( source_folder, dest_folder, now_millis, reporter, scan_wrap_if_necessary(policies_manager), encryption_wrap_if_necessary(encryption_settings_provider) ) # override to retain a public method def make_file_sync_actions( self, sync_type, source_file, dest_file, source_folder, dest_folder, now_millis, encryption_settings_provider: v2.AbstractSyncEncryptionSettingsProvider = v2. SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ Yields the sequence of actions needed to sync the two files :param str sync_type: synchronization type :param b2sdk.v1.File source_file: source file object :param b2sdk.v1.File dest_file: destination file object :param b2sdk.v1.AbstractFolder source_folder: a source folder object :param b2sdk.v1.AbstractFolder dest_folder: a destination folder object :param int now_millis: current time in milliseconds :param b2sdk.v1.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ dest_path, source_path = make_paths_from_files(dest_file, source_file, sync_type) return self._make_file_sync_actions( sync_type, source_path, dest_path, source_folder, dest_folder, now_millis, encryption_wrap_if_necessary(encryption_settings_provider), ) # override to raise old style DestFileNewer exceptions def _make_file_sync_actions( self, sync_type, source_path, dest_path, source_folder, dest_folder, now_millis, encryption_settings_provider: v2.AbstractSyncEncryptionSettingsProvider = v2. SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ Yields the sequence of actions needed to sync the two files :param str sync_type: synchronization type :param b2sdk.v1.AbstractSyncPath source_path: source file object :param b2sdk.v1.AbstractSyncPath dest_path: destination file object :param b2sdk.v1.AbstractFolder source_folder: a source folder object :param b2sdk.v1.AbstractFolder dest_folder: a destination folder object :param int now_millis: current time in milliseconds :param b2sdk.v1.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ try: yield from super()._make_file_sync_actions( sync_type, source_path, dest_path, source_folder, dest_folder, now_millis, encryption_wrap_if_necessary(encryption_settings_provider), ) except v2_exception.DestFileNewer as ex: dest_file, source_file = make_files_from_paths(ex.dest_path, ex.source_path, sync_type) raise DestFileNewer(dest_file, source_file, ex.dest_prefix, ex.source_prefix) b2-sdk-python-1.17.3/b2sdk/v2/000077500000000000000000000000001426424117700155505ustar00rootroot00000000000000b2-sdk-python-1.17.3/b2sdk/v2/__init__.py000066400000000000000000000012561426424117700176650ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk._v3 import * # noqa from b2sdk._v3 import parse_folder as parse_sync_folder from b2sdk._v3 import AbstractPath as AbstractSyncPath from b2sdk._v3 import LocalPath as LocalSyncPath from .api import B2Api from .b2http import B2Http from .bucket import Bucket, BucketFactory from .session import B2Session from .sync import B2SyncPath from .transfer import DownloadManager, UploadManager b2-sdk-python-1.17.3/b2sdk/v2/api.py000066400000000000000000000027521426424117700167010ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk import _v3 as v3 from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound from .bucket import Bucket, BucketFactory from .exception import BucketIdNotFound from .session import B2Session from .transfer import DownloadManager, UploadManager class Services(v3.Services): UPLOAD_MANAGER_CLASS = staticmethod(UploadManager) DOWNLOAD_MANAGER_CLASS = staticmethod(DownloadManager) # override to use legacy B2Session with legacy B2Http # and to raise old style BucketIdNotFound exception # and to use old style Bucket class B2Api(v3.B2Api): SESSION_CLASS = staticmethod(B2Session) BUCKET_CLASS = staticmethod(Bucket) BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) SERVICES_CLASS = staticmethod(Services) # Legacy init in case something depends on max_workers defaults = 10 def __init__(self, *args, **kwargs): kwargs.setdefault('max_upload_workers', 10) kwargs.setdefault('max_copy_workers', 10) super().__init__(*args, **kwargs) def get_bucket_by_id(self, bucket_id: str) -> v3.Bucket: try: return super().get_bucket_by_id(bucket_id) except v3BucketIdNotFound as e: raise BucketIdNotFound(e.bucket_id) b2-sdk-python-1.17.3/b2sdk/v2/b2http.py000066400000000000000000000014001426424117700173200ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/b2http.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk import _v3 as v3 from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound from .exception import BucketIdNotFound # Overridden to retain old-style BadRequest exception in case of a bad bucket id class B2Http(v3.B2Http): @classmethod def _translate_error(cls, fcn, post_params=None): try: return super()._translate_errors(fcn, post_params) except v3BucketIdNotFound as e: raise BucketIdNotFound(e.bucket_id) b2-sdk-python-1.17.3/b2sdk/v2/bucket.py000066400000000000000000000014621426424117700174020ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/bucket.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk import _v3 as v3 from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound from .exception import BucketIdNotFound # Overridden to raise old style BucketIdNotFound exception class Bucket(v3.Bucket): def get_fresh_state(self) -> 'Bucket': try: return super().get_fresh_state() except v3BucketIdNotFound as e: raise BucketIdNotFound(e.bucket_id) # Overridden to use old style Bucket class BucketFactory(v3.BucketFactory): BUCKET_CLASS = staticmethod(Bucket) b2-sdk-python-1.17.3/b2sdk/v2/exception.py000066400000000000000000000014161426424117700201220ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/exception.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk._v3.exception import * # noqa v3BucketIdNotFound = BucketIdNotFound UnSyncableFilename = UnsupportedFilename # overridden to retain old style isinstance check and attributes class BucketIdNotFound(v3BucketIdNotFound, BadRequest): def __init__(self, bucket_id): super().__init__(bucket_id) self.message = 'Bucket with id=%s not found' % (bucket_id,) self.code = 'bad_bucket_id' def __str__(self): return super(BadRequest, self).__str__() b2-sdk-python-1.17.3/b2sdk/v2/session.py000066400000000000000000000006771426424117700176170ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/session.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk import _v3 as v3 from .b2http import B2Http # Override to use legacy B2Http class B2Session(v3.B2Session): B2HTTP_CLASS = staticmethod(B2Http) b2-sdk-python-1.17.3/b2sdk/v2/sync.py000066400000000000000000000005161426424117700171000ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/sync.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk._v3 import B2Path B2SyncPath = B2Path b2-sdk-python-1.17.3/b2sdk/v2/transfer.py000066400000000000000000000031431426424117700177470ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/transfer.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from concurrent.futures import Future, ThreadPoolExecutor from typing import Callable, Optional from b2sdk import _v3 as v3 class LazyThreadPool: """ Lazily initialized thread pool. """ def __init__(self, max_workers: Optional[int] = None, **kwargs): self._max_workers = max_workers self._thread_pool = None # type: 'Optional[ThreadPoolExecutor]' super().__init__(**kwargs) def submit(self, fn: Callable, *args, **kwargs) -> Future: if self._thread_pool is None: self._thread_pool = ThreadPoolExecutor(self._max_workers) return self._thread_pool.submit(fn, *args, **kwargs) def set_size(self, max_workers: int) -> None: if self._max_workers == max_workers: return if self._thread_pool is not None: raise RuntimeError('Thread pool already created') self._max_workers = max_workers class ThreadPoolMixin(v3.ThreadPoolMixin): DEFAULT_THREAD_POOL_CLASS = staticmethod(LazyThreadPool) # This method is used in CLI even though it doesn't belong to the public API def set_thread_pool_size(self, max_workers: int) -> None: self._thread_pool.set_size(max_workers) class DownloadManager(v3.DownloadManager, ThreadPoolMixin): pass class UploadManager(v3.UploadManager, ThreadPoolMixin): pass b2-sdk-python-1.17.3/b2sdk/version.py000066400000000000000000000011471426424117700172630ustar00rootroot00000000000000###################################################################### # # File: b2sdk/version.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import sys try: from importlib.metadata import version except ModuleNotFoundError: from importlib_metadata import version # for python 3.7 VERSION = version('b2sdk') PYTHON_VERSION = '.'.join(map(str, sys.version_info[:3])) # something like: 3.9.1 USER_AGENT = 'backblaze-b2/%s python/%s' % (VERSION, PYTHON_VERSION) b2-sdk-python-1.17.3/b2sdk/version_utils.py000066400000000000000000000155671426424117700205160ustar00rootroot00000000000000###################################################################### # # File: b2sdk/version_utils.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import ABCMeta, abstractmethod from functools import wraps import inspect import warnings from pkg_resources import parse_version from b2sdk.version import VERSION class AbstractVersionDecorator(metaclass=ABCMeta): WHAT = NotImplemented # 'function', 'method', 'class' etc def __init__(self, changed_version, cutoff_version=None, reason='', current_version=None): """ Changed_version, cutoff_version and current_version are version strings. """ if current_version is None: # this is for tests only current_version = VERSION # TODO autodetect by going up the qualname tree and trying getattr(part, '__version__') self.current_version = parse_version(current_version) #: current version self.reason = reason self.changed_version = parse_version( changed_version ) #: version in which the decorator was added self.cutoff_version = self._parse_if_not_none( cutoff_version ) #: version in which the decorator (and something?) shall be removed @classmethod def _parse_if_not_none(cls, version): if version is None: return None return parse_version(version) @abstractmethod def __call__(self, func): """ The actual implementation of decorator. Needs self.source to be set before it's called. """ if self.cutoff_version is not None: assert self.changed_version < self.cutoff_version, '%s decorator is set to start renaming %s %r starting at version %s and finishing in %s. It needs to start at a lower version and finish at a higher version.' % ( self.__class__.__name__, self.WHAT, self.source, self.changed_version, self.cutoff_version, ) assert self.current_version < self.cutoff_version, '%s decorator is still used in version %s when old %s name %r was scheduled to be dropped in %s. It is time to remove the mapping.' % ( self.__class__.__name__, self.current_version, self.WHAT, self.source, self.cutoff_version, ) class AbstractDeprecator(AbstractVersionDecorator): ALTERNATIVE_DECORATOR = NotImplemented def __init__(self, target, *args, **kwargs): self.target = target super(AbstractDeprecator, self).__init__(*args, **kwargs) @abstractmethod def __call__(self, func): super(AbstractDeprecator, self).__call__(func) assert self.changed_version <= self.current_version, '%s decorator indicates that the replacement of %s %r should take place in the future version %s, while the current version is %s. It looks like should be _discouraged_ at this point and not _deprecated_ yet. Consider using %r decorator instead.' % ( self.__class__.__name__, self.WHAT, self.source, self.changed_version, self.cutoff_version, self.ALTERNATIVE_DECORATOR, ) class rename_argument(AbstractDeprecator): """ Change the argument name to new one if old one is used, warns about deprecation in docs and through a warning. >>> @rename_argument('aaa', 'bbb', '0.1.0', '0.2.0') >>> def easy(bbb): >>> return bbb >>> easy(aaa=5) 'aaa' is a deprecated argument for 'easy' function/method - it was renamed to 'bbb' in version 0.1.0. Support for the old name is going to be dropped in 0.2.0. 5 >>> """ WHAT = 'argument' ALTERNATIVE_DECORATOR = 'discourage_argument' def __init__(self, source, *args, **kwargs): self.source = source super(rename_argument, self).__init__(*args, **kwargs) def __call__(self, func): super(rename_argument, self).__call__(func) @wraps(func) def wrapper(*args, **kwargs): message = '%r is not an argument of the decorated function so it cannot be remapped to from a deprecated parameter name' % ( self.target, ) signature = inspect.getfullargspec(func) assert self.target in signature.args or self.target in signature.kwonlyargs, message if self.source in kwargs: assert self.target not in kwargs, 'both argument names were provided: %r (deprecated) and %r (new)' % ( self.source, self.target ) kwargs[self.target] = kwargs[self.source] del kwargs[self.source] warnings.warn( '%r is a deprecated argument for %r function/method - it was renamed to %r in version %s. Support for the old name is going to be dropped in %s.' % ( self.source, func.__name__, self.target, self.changed_version, self.cutoff_version, ), DeprecationWarning, ) return func(*args, **kwargs) return wrapper class rename_function(AbstractDeprecator): """ Warn about deprecation in docs and through a DeprecationWarning when used. Use it to decorate a proxy function, like this: >>> def new(foobar): >>> return foobar ** 2 >>> @rename_function(new, '0.1.0', '0.2.0') >>> def old(foo, bar): >>> return new(foo + bar) >>> old() 'old' is deprecated since version 0.1.0 - it was moved to 'new', please switch to use that. The proxy for the old name is going to be removed in 0.2.0. 123 >>> """ WHAT = 'function' ALTERNATIVE_DECORATOR = 'discourage_function' def __init__(self, target, *args, **kwargs): if callable(target): target = target.__name__ super(rename_function, self).__init__(target, *args, **kwargs) def __call__(self, func): self.source = func.__name__ super(rename_function, self).__call__(func) @wraps(func) def wrapper(*args, **kwargs): warnings.warn( '%r is deprecated since version %s - it was moved to %r, please switch to use that. The proxy for the old name is going to be removed in %s.' % ( func.__name__, self.changed_version, self.target, self.cutoff_version, ), DeprecationWarning, ) return func(*args, **kwargs) return wrapper class rename_method(rename_function): WHAT = 'method' ALTERNATIVE_DECORATOR = 'discourage_method' b2-sdk-python-1.17.3/contrib/000077500000000000000000000000001426424117700156545ustar00rootroot00000000000000b2-sdk-python-1.17.3/contrib/color-b2-logs.sh000077500000000000000000000007461426424117700206030ustar00rootroot00000000000000#!/bin/bash -eu awk -F '\t' '{print $1 " " $4 " " $5 " " $6}' | colorex --green=DEBUG \ --bgreen=INFO \ --bred=ERROR \ --byellow=WARNING \ --bmagenta='calling [\w\.]+' \ --bblue='INFO // =+ [0-9\.]+ =+ \\' \ --bblue='INFO // =+ [0-9\.]+ =+ \\' \ --bblue='starting command .* with arguments:' \ --bblue='starting command .* \(arguments hidden\)' \ --red=Traceback \ --green='\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d' \ --cyan='b2\.sync' b2-sdk-python-1.17.3/contrib/debug_logs.ini000066400000000000000000000013071426424117700204700ustar00rootroot00000000000000############################################################ [loggers] keys=root,b2 [logger_root] level=DEBUG handlers=fileHandler [logger_b2] level=DEBUG handlers=fileHandler qualname=b2 propagate=0 ############################################################ [handlers] keys=fileHandler [handler_fileHandler] class=logging.handlers.TimedRotatingFileHandler level=DEBUG formatter=simpleFormatter args=('b2sdk.log', 'midnight') ############################################################ [formatters] keys=simpleFormatter [formatter_simpleFormatter] format=%(asctime)s %(process)d %(thread)d %(name)s %(levelname)s %(message)s datefmt= ############################################################ b2-sdk-python-1.17.3/doc/000077500000000000000000000000001426424117700147615ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/bash_completion.md000066400000000000000000000021561426424117700204550ustar00rootroot00000000000000In order to use bash completion, you can have a `~/.bash_completion` like this: ```sh if [ -d "$HOME/.bash_completion.d" ]; then for file in "$HOME/.bash_completion.d/"* do source "$file" >/dev/null 2>&1 done fi ``` and then copy our `contrib/bash_completion/b2` to your `~/.bash_completion.d/`. The important trick is that `b2` tool must be in PATH before bash_completions are loaded for the last time (unless you delete the first line of our completion script). If you keep the `b2` tool in `~/bin`, you can make sure the loading order is proper by making sure `~/bin` is added to the PATH before loading bash_completion. To do that, add the following snippet to your `~/.bashrc`: ```sh if [ -d ~/bin ]; then PATH="$HOME/bin:$PATH" fi # enable programmable completion features (you don't need to enable # this, if it's already enabled in /etc/bash.bashrc and /etc/profile # sources /etc/bash.bashrc). if ! shopt -oq posix; then if [ -f /usr/share/bash-completion/bash_completion ]; then . /usr/share/bash-completion/bash_completion elif [ -f /etc/bash_completion ]; then . /etc/bash_completion fi fi ``` b2-sdk-python-1.17.3/doc/markup-test.rst000066400000000000000000000107271426424117700177760ustar00rootroot00000000000000Test ~~~~ .. DANGER:: Beware killer rabbits! .. note:: This is a note admonition. This is the second line of the first paragraph. - The note contains all indented body elements following. - It includes this bullet list. .. admonition:: And, by the way... You can make up your own admonition too. .. title:: This is title block of text .. admonition:: This is admonition block of text .. warning:: This is warning block of text .. tip:: This is tip block of text .. note:: This is note block of text .. important:: This is important block of text .. hint:: This is hint block of text .. error:: This is error block of text .. danger:: This is danger block of text .. caution:: This is caution block of text .. attention:: This is attention block of text .. _semver: Types of interfaces and version pinning ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ b2sdk is divided into three parts. Please pay attention to which group you use, as the stability of your application depends on correct pinning of versions. Public ~~~~~~ Public interface consists of *public* members of the following modules: * b2sdk.api.B2Api * b2sdk.bucket.Bucket * b2sdk.exception * b2sdk.sync * b2sdk.sync.exception and some of their dependencies: * b2sdk.account_info.InMemoryAccountInfo * b2sdk.account_info.SqliteAccountInfo * b2sdk.transferer * b2sdk.utils Those will not change in a backwards-incompatible way between non-major versions. In other words, if you pin your dependencies to `>=x.0.0;=4.5.6;<5.0.0 .. note:: b2sdk.*._something and b2sdk.*.*._something, having a name which begins with an underscore, are NOT considered public interface. Protected ~~~~~~~~~ Things which sometimes might be necessary to use that are NOT considered public interface (and may change in a non-major version): * B2Session * B2RawHTTPApi * B2Http .. note:: it is ok for you to use those (better that, than copying our sources), however if you do, please pin your dependencies to middle version. .. example:: If the current version of b2sdk is 4.5.6 and you use the public and protected interfaces, put this in your requirements.txt: >=4.5.6;<4.6.0 Private ~~~~~~~ If you need to use some of our private interfaces, pin your dependencies strictly. .. example:: If the current version of b2sdk is 4.5.6 and you use the private interface, put this in your requirements.txt: ==4.5.6 Authorization ~~~~~~~~~~~~~ Before you can use b2sdk, you need to prove who you are to the server. For that you will need to pass `account id` and `api token` to one of the authorization classes. In case you are storing that information in a database or something, you can implement your own class by inheriting from AbstractAuthorization. Otherwise, use one of the classes included in b2sdk package: InMemoryAccountInfo: This is probably what your application should be using and also what we use in our tests. SqliteAccountInfo: this is what B2 CLI uses to authorize the user. Stores information in a local file. B2Api ~~~~ The "main" object that abstracts the communication with B2 cloud is B2Api. It lets you manage buckets and download files by id. example Bucket ~~~~~~ Bucket class abstracts the B2 bucket, which is essentially a namespace for objects. The best way to transfer your files into a bucket and back, is to use *sync*. If for some reason you cannot use sync, it is also possible to upload and download files directly into/from the bucket, using Bucket.upload_file and Bucket.download_by_name. The Bucket object also contains a few methods to list the contents of the bucket and the metadata associated with the objects contained in it. ======== Tutorial ======== Account authorization ===================== TODO Bucket actions ============== Create a bucket --------------- TODO Remove a bucket --------------- TODO List buckets ------------- TODO Update bucket info ------------------ TODO File actions ============ Upload file ----------- TODO Download file ------------- TODO List files ---------- TODO Get file meta information ------------------------- TODO Delete file ----------- TODO Cancel file operations ---------------------- TODO Synchronization =============== TODO Account information =================== TODO b2-sdk-python-1.17.3/doc/render_sqlite_account_info_schema.sh000077500000000000000000000001241426424117700242240ustar00rootroot00000000000000python ../sqlite_account_info_schema.py > source/dot/sqlite_account_info_schema.dot b2-sdk-python-1.17.3/doc/source/000077500000000000000000000000001426424117700162615ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/advanced.rst000066400000000000000000000504631426424117700205700ustar00rootroot00000000000000.. _AdvancedUsagePatterns: ######################################### Advanced usage patterns ######################################### B2 server API allows for creation of an object from existing objects. This allows to avoid transferring data from the source machine if the desired outcome can be (at least partially) constructed from what is already on the server. The way **b2sdk** exposes this functionality is through a few functions that allow the user to express the desired outcome and then the library takes care of planning and executing the work. Please refer to the table below to compare the support of object creation methods for various usage patterns. ***************** Available methods ***************** .. |br| raw:: html
.. _advanced_methods_support_table: +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | Method / supported options | Source | Range |br| overlap | Streaming |br| interface | :ref:`Continuation ` | +============================================+========+=====================+==========================+====================================+ | :meth:`b2sdk.v2.Bucket.upload` | local | no | no | automatic | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | :meth:`b2sdk.v2.Bucket.copy` | remote | no | no | automatic | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | :meth:`b2sdk.v2.Bucket.concatenate` | any | no | no | automatic | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | :meth:`b2sdk.v2.Bucket.concatenate_stream` | any | no | yes | manual | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | :meth:`b2sdk.v2.Bucket.create_file` | any | yes | no | automatic | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | :meth:`b2sdk.v2.Bucket.create_file_stream` | any | yes | yes | manual | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ Range overlap ============= Some methods support overlapping ranges between local and remote files. **b2sdk** tries to use the remote ranges as much as possible, but due to limitations of ``b2_copy_part`` (specifically the minimum size of a part) that may not be always possible. A possible solution for such case is to download a (small) range and then upload it along with another one, to meet the ``b2_copy_part`` requirements. This can be improved if the same data is already available locally - in such case **b2sdk** will use the local range rather than downloading it. Streaming interface =================== Some object creation methods start writing data before reading the whole input (iterator). This can be used to write objects that do not have fully known contents without writing them first locally, so that they could be copied. Such usage pattern can be relevant to small devices which stream data to B2 from an external NAS, where caching large files such as media files or virtual machine images is not an option. Please see :ref:`advanced method support table ` to see where streaming interface is supported. Continuation ============ Please see :ref:`here ` ***************** Concatenate files ***************** :meth:`b2sdk.v2.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). It can be used to glue remote files together, back-to-back, into a new file. :meth:`b2sdk.v2.Bucket.concatenate_stream` does not create and validate a plan before starting the transfer, so it can be used to process a large input iterator, at a cost of limited automated continuation. Concatenate files of known size ================================= .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> input_sources = [ ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, length=100), ... UploadSourceLocalFile('my_local_path/to_file.txt'), ... CopySource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.concatenate(input_sources, remote_name, file_info) If one of remote source has length smaller than :term:`absoluteMinimumPartSize` then it cannot be copied into large file part. Such remote source would be downloaded and concatenated locally with local source or with other downloaded remote source. Please note that this method only allows checksum verification for local upload sources. Checksum verification for remote sources is available only when local copy is available. In such case :meth:`b2sdk.v2.Bucket.create_file` can be used with overalapping ranges in input. For more information about ``concatenate`` please see :meth:`b2sdk.v2.Bucket.concatenate` and :class:`b2sdk.v2.CopySource`. Concatenate files of known size (streamed version) ================================================== :meth:`b2sdk.v2.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). The operation would not be planned ahead so it supports very large output objects, but continuation is only possible for local only sources and provided unfinished large file id. See more about continuation in :meth:`b2sdk.v2.Bucket.create_file` paragraph about continuation. .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> input_sources = [ ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, length=100), ... UploadSourceLocalFile('my_local_path/to_file.txt'), ... CopySource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.concatenate_stream(input_sources, remote_name, file_info) Concatenate files of unknown size ================================= While it is supported by B2 server, this pattern is currently not supported by **b2sdk**. ********************* Synthethize an object ********************* Using methods described below an object can be created from both local and remote sources while avoiding downloading small ranges when such range is already present on a local drive. Update a file efficiently ==================================== :meth:`b2sdk.v2.Bucket.create_file` accepts an iterable which *can contain overlapping destination ranges*. .. note:: Following examples *create* new file - data in bucket is immutable, but **b2sdk** can create a new file version with the same name and updated content Append to the end of a file --------------------------- The assumption here is that the file has been appended to since it was last uploaded to. This assumption is verified by **b2sdk** when possible by recalculating checksums of the overlapping remote and local ranges. If copied remote part sha does not match with locally available source, file creation process would be interrupted and an exception would be raised. .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> input_sources = [ ... WriteIntent( ... data=CopySource( ... '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', ... offset=0, ... length=5000000, ... ), ... destination_offset=0, ... ), ... WriteIntent( ... data=UploadSourceLocalFile('my_local_path/to_file.txt'), # of length 60000000 ... destination_offset=0, ... ), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.create_file(input_sources, remote_name, file_info) `LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v2.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range (when concatenating, local source would have destination offset at the end of remote source) For more information see :meth:`b2sdk.v2.Bucket.create_file`. Change the middle of the remote file ------------------------------------ .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> input_sources = [ ... WriteIntent( ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=4000000), ... destination_offset=0, ... ), ... WriteIntent( ... UploadSourceLocalFile('my_local_path/to_file.txt'), # length=1024, here not passed and just checked from local source using seek ... destination_offset=4000000, ... ), ... WriteIntent( ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=4001024, length=123456789), ... destination_offset=4001024, ... ), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.create_file(input_sources, remote_name, file_info) `LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v2.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range. For more information see :meth:`b2sdk.v2.Bucket.create_file`. Synthetize a file from local and remote parts ============================================= This is useful for expert usage patterns such as: - *synthetic backup* - *reverse synthetic backup* - mostly-server-side cutting and gluing uncompressed media files such as `wav` and `avi` with rewriting of file headers - various deduplicated backup scenarios Please note that :meth:`b2sdk.v2.Bucket.create_file_stream` accepts **an ordered iterable** which *can contain overlapping ranges*, so the operation does not need to be planned ahead, but can be streamed, which supports very large output objects. Scenarios such as below are then possible: .. code-block:: A C D G | | | | | cloud-AC | | cloud-DG | | | | | v v v v ############ ############# ^ ^ | | +---- desired file A-G --------+ | | | | | ######################### | | ^ ^ | | | | | | | local file-BF | | | | | | A B C D E F G .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> def generate_input(): ... yield WriteIntent( ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=lengthC), ... destination_offset=0, ... ) ... yield WriteIntent( ... UploadSourceLocalFile('my_local_path/to_file.txt'), # length = offsetF - offsetB ... destination_offset=offsetB, ... ) ... yield WriteIntent( ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=offsetG-offsetD), ... destination_offset=offsetD, ... ) ... >>> file_info = {'how': 'good-file'} >>> bucket.create_file(generate_input(), remote_name, file_info) In such case, if the sizes allow for it (there would be no parts smaller than :term:`absoluteMinimumPartSize`), the only uploaded part will be `C-D`. Otherwise, more data will be uploaded, but the data transfer will be reduced in most cases. :meth:`b2sdk.v2.Bucket.create_file` does not guarantee that outbound transfer usage would be optimal, it uses a simple greedy algorithm with as small look-aheads as possible. For more information see :meth:`b2sdk.v2.Bucket.create_file`. Encryption ---------- Even if files `A-C` and `D-G` are encrypted using `SSE-C` with different keys, they can still be used in a single :meth:`b2sdk.v2.Bucket.create_file` call, because :class:`b2sdk.v2.CopySource` accepts an optional :class:`b2sdk.v2.EncryptionSetting`. Prioritize remote or local sources ---------------------------------- :meth:`b2sdk.v2.Bucket.create_file` and :meth:`b2sdk.v2.Bucket.create_file_stream` support source/origin prioritization, so that planner would know which sources should be used for overlapping ranges. Supported values are: `local`, `remote` and `local_verification`. .. code-block:: A D G | | | | cloud-AD | | | | | v v | ################ | ^ | | | +---- desired file A-G --------+ | | | | | ####### ################# | ^ ^ ^ | | | | | | | | local file BC and DE | | | | | | A B C D E A=0, B=50M, C=80M, D=100M, E=200 .. code-block:: python >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local') # planner parts: cloud[A, B], local[B, C], remote[C, D], local[D, E] Here the planner has only used a remote source where remote range was not available, minimizing downloads. .. code-block:: python >>> planner.create_file(input_sources, remote_name, file_info, prioritize='remote') # planner parts: cloud[A, D], local[D, E] Here the planner has only used a local source where remote range was not available, minimizing uploads. .. code-block:: python >>> bucket.create_file(input_sources, remote_name, file_info) # or >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local_verification') # planner parts: cloud[A, B], cloud[B, C], cloud[C, D], local[D, E] In `local_verification` mode the remote range was artificially split into three parts to allow for checksum verification against matching local ranges. .. note:: `prioritize` is just a planner setting - remote parts are always verified if matching local parts exists. .. TODO:: prioritization should accept enum, not string .. _continuation: ************ Continuation ************ Continuation of upload ====================== In order to continue a simple upload session, **b2sdk** checks for any available sessions with of the same ``file name``, ``file_info`` and ``media type``, verifying the size of an object as much as possible. To support automatic continuation, some advanced methods create a plan before starting copy/upload operations, saving the hash of that plan in ``file_info`` for increased reliability. If that is not available, ``large_file_id`` can be extracted via callback during the operation start. It can then be passed into the subsequent call to continue the same task, though the responsibility for passing the exact same input is then on the user of the function. Please see :ref:`advanced method support table ` to see where automatic continuation is supported. ``large_file_id`` can also be passed if automatic continuation is available in order to avoid issues where multiple matching upload sessions are matching the transfer. Continuation of create/concantenate =================================== :meth:`b2sdk.v2.Bucket.create_file` supports automatic continuation or manual continuation. :meth:`b2sdk.v2.Bucket.create_file_stream` supports only manual continuation for local-only inputs. The situation looks the same for :meth:`b2sdk.v2.Bucket.concatenate` and :meth:`b2sdk.v2.Bucket.concatenate_stream` (streamed version supports only manual continuation of local sources). Also :meth:`b2sdk.v2.Bucket.upload` and :meth:`b2sdk.v2.Bucket.copy` support both automatic and manual continuation. Manual continuation ------------------- .. code-block:: python >>> def large_file_callback(large_file): ... # storage is not part of the interface - here only for demonstration purposes ... storage.store({'name': remote_name, 'large_file_id': large_file.id}) >>> bucket.create_file(input_sources, remote_name, file_info, large_file_callback=large_file_callback) # ... >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] >>> bucket.create_file(input_sources, remote_name, file_info, large_file_id=large_file_id) Manual continuation (streamed version) -------------------------------------- .. code-block:: python >>> def large_file_callback(large_file): ... # storage is not part of the interface - here only for demonstration purposes ... storage.store({'name': remote_name, 'large_file_id': large_file.id}) >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_callback=large_file_callback) # ... >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_id=large_file_id) Streams that contains remote sources cannot be continued with :meth:`b2sdk.v2.Bucket.create_file` - internally :meth:`b2sdk.v2.Bucket.create_file` stores plan information in file info for such inputs, and verifies it before any copy/upload and :meth:`b2sdk.v2.Bucket.create_file_stream` cannot store this information. Local source only inputs can be safely continued with :meth:`b2sdk.v2.Bucket.create_file` in auto continue mode or manual continue mode (because plan information is not stored in file info in such case). Auto continuation ----------------- .. code-block:: python >>> bucket.create_file(input_sources, remote_name, file_info) For local source only input, :meth:`b2sdk.v2.Bucket.create_file` would try to find matching unfinished large file. It will verify uploaded parts checksums with local sources - the most completed, having all uploaded parts matched candidate would be automatically selected as file to continue. If there is no matching candidate (even if there are unfinished files for the same file name) new large file would be started. In other cases plan information would be generated and :meth:`b2sdk.v2.Bucket.create_file` would try to find unfinished large file with matching plan info in its file info. If there is one or more such unfinished large files, :meth:`b2sdk.v2.Bucket.create_file` would verify checksums for all locally available parts and choose any matching candidate. If all candidates fails on uploaded parts checksums verification, process is interrupted and error raises. In such case corrupted unfinished large files should be cancelled manullay and :meth:`b2sdk.v2.Bucket.create_file` should be retried, or auto continuation should be turned off with `auto_continue=False` No continuation --------------- .. code-block:: python >>> bucket.create_file(input_sources, remote_name, file_info, auto_continue=False) Note, that this only forces start of a new large file - it is still possible to continue the process with either auto or manual modes. b2-sdk-python-1.17.3/doc/source/api/000077500000000000000000000000001426424117700170325ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/account_info.rst000066400000000000000000000104611426424117700222350ustar00rootroot00000000000000.. _AccountInfo: ######################## AccountInfo ######################## *AccountInfo* stores basic information about the account, such as *Application Key ID* and *Application Key*, in order to let :py:class:`b2sdk.v2.B2Api` perform authenticated requests. There are two usable implementations provided by **b2sdk**: * :py:class:`b2sdk.v2.InMemoryAccountInfo` - a basic implementation with no persistence * :py:class:`b2sdk.v2.SqliteAccountInfo` - for console and GUI applications They both provide the full :ref:`AccountInfo interface `. .. note:: Backup applications and many server-side applications should :ref:`implement their own ` *AccountInfo*, backed by the metadata/configuration database of the application. *************************** AccountInfo implementations *************************** InMemoryAccountInfo =================== *AccountInfo* with no persistence. .. autoclass:: b2sdk.v2.InMemoryAccountInfo() :no-members: Implements all methods of :ref:`AccountInfo interface `. .. hint:: Usage of this class is appropriate for secure Web applications which do not wish to persist any user data. Using this class for applications such as CLI, GUI or backup is discouraged, as ``InMemoryAccountInfo`` does not write down the authorization token persistently. That would be slow, as it would force the application to retrieve a new one on every command/click/backup start. Furthermore - an important property of *AccountInfo* is caching the ``bucket_name:bucket_id`` mapping; in case of ``InMemoryAccountInfo`` the cache will be flushed between executions of the program. .. method:: __init__() The constructor takes no parameters. SqliteAccountInfo ================= .. autoclass:: b2sdk.v2.SqliteAccountInfo() :inherited-members: :no-members: :special-members: __init__ Implements all methods of :ref:`AccountInfo interface `. Uses a `SQLite database `_ for persistence and access synchronization between multiple processes. Not suitable for usage over NFS. Underlying database has the following schema: .. graphviz:: /dot/sqlite_account_info_schema.dot .. hint:: Usage of this class is appropriate for interactive applications installed on a user's machine (i.e.: CLI and GUI applications). Usage of this class **might** be appropriate for non-interactive applications installed on the user's machine, such as backup applications. An alternative approach that should be considered is to store the *AccountInfo* data alongside the configuration of the rest of the application. .. _my_account_info: ********************* Implementing your own ********************* When building a server-side application or a web service, you might want to implement your own *AccountInfo* class backed by a database. In such case, you should inherit from :py:class:`b2sdk.v2.UrlPoolAccountInfo`, which has groundwork for url pool functionality). If you cannot use it, inherit directly from :py:class:`b2sdk.v2.AbstractAccountInfo`. .. code-block:: python >>> from b2sdk.v2 import UrlPoolAccountInfo >>> class MyAccountInfo(UrlPoolAccountInfo): ... :py:class:`b2sdk.v2.AbstractAccountInfo` describes the interface, while :py:class:`b2sdk.v2.UrlPoolAccountInfo` and :py:class:`b2sdk.v2.UploadUrlPool` implement a part of the interface for in-memory upload token management. .. _account_info_interface: AccountInfo interface ===================== .. autoclass:: b2sdk.v2.AbstractAccountInfo() :inherited-members: :private-members: :exclude-members: _abc_cache, _abc_negative_cache, _abc_negative_cache_version, _abc_registry AccountInfo helper classes ========================== .. autoclass:: b2sdk.v2.UrlPoolAccountInfo() :inherited-members: :no-members: :members: BUCKET_UPLOAD_POOL_CLASS, LARGE_FILE_UPLOAD_POOL_CLASS .. caution:: This class is not part of the public interface. To find out how to safely use it, read :ref:`this `. .. autoclass:: b2sdk.account_info.upload_url_pool.UploadUrlPool() :inherited-members: :private-members: .. caution:: This class is not part of the public interface. To find out how to safely use it, read :ref:`this `. b2-sdk-python-1.17.3/doc/source/api/api.rst000066400000000000000000000003711426424117700203360ustar00rootroot00000000000000B2 Api client =============================================== .. autoclass:: b2sdk.v2.B2Api() :inherited-members: :special-members: __init__ .. autoclass:: b2sdk.v2.B2HttpApiConfig() :inherited-members: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/application_key.rst000066400000000000000000000003551426424117700227420ustar00rootroot00000000000000B2 Application key ================== .. autoclass:: b2sdk.v2.ApplicationKey() :inherited-members: :special-members: __init__ .. autoclass:: b2sdk.v2.FullApplicationKey() :inherited-members: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/bucket.rst000066400000000000000000000002231426424117700210360ustar00rootroot00000000000000B2 Bucket =============================================== .. autoclass:: b2sdk.v2.Bucket() :inherited-members: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/cache.rst000066400000000000000000000010441426424117700206260ustar00rootroot00000000000000Cache =============================================== **b2sdk** caches the mapping between bucket name and bucket id, so that the user of the library does not need to maintain the mapping to call the api. .. autoclass:: b2sdk.v2.AbstractCache :inherited-members: .. autoclass:: b2sdk.v2.AuthInfoCache() :inherited-members: :special-members: __init__ .. autoclass:: b2sdk.v2.DummyCache() :inherited-members: :special-members: __init__ .. autoclass:: b2sdk.v2.InMemoryCache() :inherited-members: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/data_classes.rst000066400000000000000000000007551426424117700222210ustar00rootroot00000000000000Data classes =============================================== .. autoclass:: b2sdk.v2.FileVersion :inherited-members: :special-members: __dict__ .. autoclass:: b2sdk.v2.DownloadVersion :inherited-members: .. autoclass:: b2sdk.v2.FileIdAndName :inherited-members: :special-members: __dict__ .. autoclass:: b2sdk.v2.UnfinishedLargeFile() :no-members: .. autoclass:: b2sdk.v2.Part :no-members: .. autoclass:: b2sdk.v2.Range :no-members: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/downloaded_file.rst000066400000000000000000000001631426424117700227030ustar00rootroot00000000000000Downloaded File =============== .. autoclass:: b2sdk.v2.DownloadedFile .. autoclass:: b2sdk.v2.MtimeUpdatedFile b2-sdk-python-1.17.3/doc/source/api/encryption/000077500000000000000000000000001426424117700212245ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/encryption/setting.rst000066400000000000000000000007211426424117700234330ustar00rootroot00000000000000.. _encryption_setting: Encryption Settings =================== .. autoclass:: b2sdk.v2.EncryptionKey() :no-members: :special-members: __init__ .. autoclass:: b2sdk.v2.UNKNOWN_KEY_ID :no-members: .. autoclass:: b2sdk.v2.EncryptionSetting() :no-members: :special-members: __init__, as_dict .. autoattribute:: b2sdk.v2.SSE_NONE Commonly used "no encryption" setting .. autoattribute:: b2sdk.v2.SSE_B2_AES Commonly used SSE-B2 setting b2-sdk-python-1.17.3/doc/source/api/encryption/types.rst000066400000000000000000000001411426424117700231160ustar00rootroot00000000000000.. _encryption_types: Encryption Types ================ .. automodule:: b2sdk.encryption.types b2-sdk-python-1.17.3/doc/source/api/enums.rst000066400000000000000000000005031426424117700207110ustar00rootroot00000000000000Enums =============================================== .. autoclass:: b2sdk.v2.MetadataDirectiveMode :inherited-members: .. autoclass:: b2sdk.v2.NewerFileSyncMode :inherited-members: .. autoclass:: b2sdk.v2.CompareVersionMode :inherited-members: .. autoclass:: b2sdk.v2.KeepOrDeleteMode :inherited-members: b2-sdk-python-1.17.3/doc/source/api/exception.rst000066400000000000000000000003021426424117700215550ustar00rootroot00000000000000Exceptions ==================================== .. todo:: improve documentation of exceptions, automodule -> autoclass? .. automodule:: b2sdk.v2.exception :members: :undoc-members: b2-sdk-python-1.17.3/doc/source/api/file_lock.rst000066400000000000000000000015771426424117700215250ustar00rootroot00000000000000File locks ========== .. autoclass:: b2sdk.v2.LegalHold() :no-members: :special-members: ON, OFF, UNSET, UNKNOWN, is_on, is_off, is_unknown .. autoclass:: b2sdk.v2.FileRetentionSetting() :no-members: :special-members: __init__ .. autoclass:: b2sdk.v2.RetentionMode() :inherited-members: :members: .. autoclass:: b2sdk.v2.BucketRetentionSetting() :no-members: :special-members: __init__ .. autoclass:: b2sdk.v2.RetentionPeriod() :inherited-members: :special-members: __init__ .. autoclass:: b2sdk.v2.FileLockConfiguration() :no-members: :special-members: __init__ .. autoclass:: b2sdk.v2.UNKNOWN_BUCKET_RETENTION() .. autoclass:: b2sdk.v2.UNKNOWN_FILE_LOCK_CONFIGURATION() .. autoclass:: b2sdk.v2.NO_RETENTION_BUCKET_SETTING() .. autoclass:: b2sdk.v2.NO_RETENTION_FILE_SETTING() .. autoclass:: b2sdk.v2.UNKNOWN_FILE_RETENTION_SETTING() b2-sdk-python-1.17.3/doc/source/api/internal/000077500000000000000000000000001426424117700206465ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/internal/b2http.rst000066400000000000000000000001761426424117700226070ustar00rootroot00000000000000:mod:`b2sdk.b2http` -- thin http client wrapper =============================================== .. automodule:: b2sdk.b2http b2-sdk-python-1.17.3/doc/source/api/internal/cache.rst000066400000000000000000000002441426424117700224430ustar00rootroot00000000000000:mod:`b2sdk.cache` =========================== .. automodule:: b2sdk.cache :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/raw_api.rst000066400000000000000000000003201426424117700230150ustar00rootroot00000000000000:mod:`b2sdk.raw_api` -- B2 raw api wrapper ============================================= .. automodule:: b2sdk.raw_api :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/raw_simulator.rst000066400000000000000000000003431426424117700242700ustar00rootroot00000000000000:mod:`b2sdk.raw_simulator` -- B2 raw api simulator ================================================== .. automodule:: b2sdk.raw_simulator :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/requests.rst000066400000000000000000000002421426424117700232510ustar00rootroot00000000000000:mod:`b2sdk.requests` -- modified requests.models.Response class ================================================================ .. automodule:: b2sdk.requests b2-sdk-python-1.17.3/doc/source/api/internal/scan/000077500000000000000000000000001426424117700215725ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/internal/scan/folder.rst000066400000000000000000000002671426424117700236040ustar00rootroot00000000000000:mod:`b2sdk.scan.folder` ================================== .. automodule:: b2sdk.scan.folder :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/scan/folder_parser.rst000066400000000000000000000003231426424117700251510ustar00rootroot00000000000000:mod:`b2sdk.scan.folder_parser` ================================================ .. automodule:: b2sdk.scan.folder_parser :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/scan/path.rst000066400000000000000000000002571426424117700232640ustar00rootroot00000000000000:mod:`b2sdk.scan.path` ============================== .. automodule:: b2sdk.scan.path :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/scan/policies.rst000066400000000000000000000003111426424117700241260ustar00rootroot00000000000000:mod:`b2sdk.scan.policies` ================================================ .. automodule:: b2sdk.scan.policies :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/scan/scan.rst000066400000000000000000000003011426424117700232420ustar00rootroot00000000000000:mod:`b2sdk.scan.scan` ================================================ .. automodule:: b2sdk.scan.scan :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/session.rst000066400000000000000000000003101426424117700230550ustar00rootroot00000000000000:mod:`b2sdk.session` -- B2 Session ============================================= .. automodule:: b2sdk.session :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/stream/000077500000000000000000000000001426424117700221415ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/internal/stream/chained.rst000066400000000000000000000003251426424117700242660ustar00rootroot00000000000000:mod:`b2sdk.stream.chained` ChainedStream ============================================ .. automodule:: b2sdk.stream.chained :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/stream/hashing.rst000066400000000000000000000003261426424117700243150ustar00rootroot00000000000000:mod:`b2sdk.stream.hashing` StreamWithHash ============================================ .. automodule:: b2sdk.stream.hashing :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/stream/progress.rst000066400000000000000000000003711426424117700245400ustar00rootroot00000000000000:mod:`b2sdk.stream.progress` Streams with progress reporting ============================================================ .. automodule:: b2sdk.stream.progress :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/stream/range.rst000066400000000000000000000003271426424117700237710ustar00rootroot00000000000000:mod:`b2sdk.stream.range` RangeOfInputStream ============================================= .. automodule:: b2sdk.stream.range :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/stream/wrapper.rst000066400000000000000000000003221426424117700243500ustar00rootroot00000000000000:mod:`b2sdk.stream.wrapper` StreamWrapper ========================================= .. automodule:: b2sdk.stream.wrapper :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/sync/000077500000000000000000000000001426424117700216225ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/internal/sync/action.rst000066400000000000000000000002741426424117700236340ustar00rootroot00000000000000:mod:`b2sdk.sync.action` ======================================= .. automodule:: b2sdk.sync.action :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/sync/exception.rst000066400000000000000000000003111426424117700243450ustar00rootroot00000000000000:mod:`b2sdk.sync.exception` ============================================== .. automodule:: b2sdk.sync.exception :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/sync/policy.rst000066400000000000000000000002671426424117700236600ustar00rootroot00000000000000:mod:`b2sdk.sync.policy` ================================== .. automodule:: b2sdk.sync.policy :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/sync/policy_manager.rst000066400000000000000000000003271426424117700253470ustar00rootroot00000000000000:mod:`b2sdk.sync.policy_manager` ================================================== .. automodule:: b2sdk.sync.policy_manager :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/sync/sync.rst000066400000000000000000000002571426424117700233340ustar00rootroot00000000000000:mod:`b2sdk.sync.sync` ============================== .. automodule:: b2sdk.sync.sync :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/transfer/000077500000000000000000000000001426424117700224725ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/internal/transfer/inbound/000077500000000000000000000000001426424117700241305ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/internal/transfer/inbound/download_manager.rst000066400000000000000000000004431426424117700301640ustar00rootroot00000000000000:mod:`b2sdk.transfer.inbound.download_manager` -- Manager of downloaders ======================================================================== .. automodule:: b2sdk.transfer.inbound.download_manager :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/transfer/inbound/downloader/000077500000000000000000000000001426424117700262665ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/internal/transfer/inbound/downloader/abstract.rst000066400000000000000000000004521426424117700306240ustar00rootroot00000000000000:mod:`b2sdk.transfer.inbound.downloader.abstract` -- Downloader base class ========================================================================== .. automodule:: b2sdk.transfer.inbound.downloader.abstract :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/transfer/inbound/downloader/parallel.rst000066400000000000000000000004441426424117700306160ustar00rootroot00000000000000:mod:`b2sdk.transfer.inbound.downloader.parallel` -- ParallelTransferer ======================================================================= .. automodule:: b2sdk.transfer.inbound.downloader.parallel :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/transfer/inbound/downloader/simple.rst000066400000000000000000000004321426424117700303100ustar00rootroot00000000000000:mod:`b2sdk.transfer.inbound.downloader.simple` -- SimpleDownloader =================================================================== .. automodule:: b2sdk.transfer.inbound.downloader.simple :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/transfer/outbound/000077500000000000000000000000001426424117700243315ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/internal/transfer/outbound/upload_source.rst000066400000000000000000000003511426424117700277260ustar00rootroot00000000000000:mod:`b2sdk.transfer.outbound.upload_source` ============================================ .. automodule:: b2sdk.transfer.outbound.upload_source :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/internal/utils.rst000066400000000000000000000002611426424117700225370ustar00rootroot00000000000000:mod:`b2sdk.utils` ======================================== .. automodule:: b2sdk.utils :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/progress.rst000066400000000000000000000012341426424117700214300ustar00rootroot00000000000000Progress reporters ========================================== .. note:: Concrete classes described in this chapter implement methods defined in ``AbstractProgressListener`` .. todo:: improve documentation of progress reporters include info about sync progress .. autoclass:: b2sdk.v2.AbstractProgressListener :inherited-members: :members: .. autoclass:: b2sdk.v2.TqdmProgressListener :no-members: .. autoclass:: b2sdk.v2.SimpleProgressListener :no-members: .. autoclass:: b2sdk.v2.DoNothingProgressListener :no-members: .. autoclass:: b2sdk.v2.ProgressListenerForTest :no-members: .. autofunction:: b2sdk.v2.make_progress_listener b2-sdk-python-1.17.3/doc/source/api/sync.rst000066400000000000000000000203101426424117700205340ustar00rootroot00000000000000.. _sync: ######################## Synchronizer ######################## Synchronizer is a powerful utility with functionality of a basic backup application. It is able to copy entire folders into the cloud and back to a local drive or even between two cloud buckets, providing retention policies and many other options. The **high performance** of sync is credited to parallelization of: * listing local directory contents * listing bucket contents * uploads * downloads Synchronizer spawns threads to perform the operations listed above in parallel to shorten the backup window to a minimum. Sync Options ============ Following are the important optional arguments that can be provided while initializing `Synchronizer` class. * ``compare_version_mode``: When comparing the source and destination files for finding whether to replace them or not, `compare_version_mode` can be passed to specify the mode of comparison. For possible values see :class:`b2sdk.v2.CompareVersionMode`. Default value is :py:attr:`b2sdk.v2.CompareVersionMode.MODTIME` * ``compare_threshold``: It's the minimum size (in bytes)/modification time (in seconds) difference between source and destination files before we assume that it is new and replace. * ``newer_file_mode``: To identify whether to skip or replace if source is older. For possible values see :class:`b2sdk.v2.NewerFileSyncMode`. If you don't specify this the sync will raise :class:`b2sdk.v2.exception.DestFileNewer` in case any of the source file is older than destination. * ``keep_days_or_delete``: specify policy to keep or delete older files. For possible values see :class:`b2sdk.v2.KeepOrDeleteMode`. Default is `DO_NOTHING`. * ``keep_days``: if `keep_days_or_delete` is :py:attr:`b2sdk.v2.KeepOrDeleteMode.KEEP_BEFORE_DELETE` then this specifies for how many days should we keep. .. code-block:: python >>> from b2sdk.v2 import ScanPoliciesManager >>> from b2sdk.v2 import parse_folder >>> from b2sdk.v2 import Synchronizer >>> from b2sdk.v2 import KeepOrDeleteMode, CompareVersionMode, NewerFileSyncMode >>> import time >>> import sys >>> source = '/home/user1/b2_example' >>> destination = 'b2://example-mybucket-b2' >>> source = parse_folder(source, b2_api) >>> destination = parse_folder(destination, b2_api) >>> policies_manager = ScanPoliciesManager(exclude_all_symlinks=True) >>> synchronizer = Synchronizer( max_workers=10, policies_manager=policies_manager, dry_run=False, allow_empty_source=True, compare_version_mode=CompareVersionMode.SIZE, compare_threshold=10, newer_file_mode=NewerFileSyncMode.REPLACE, keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=10, ) We have a file (hello.txt) which is present in destination but not on source (my local), so it will be deleted and since our mode is to keep the delete file, it will be hidden for 10 days in bucket. .. code-block:: python >>> no_progress = False >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) upload f1.txt delete hello.txt (old version) hide hello.txt We changed f1.txt and added 1 byte. Since our compare_threshold is 10, it will not do anything. .. code-block:: python >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) We changed f1.txt and added more than 10 bytes. Since our compare_threshold is 10, it will replace the file at destination folder. .. code-block:: python >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) upload f1.txt Let's just delete the file and not keep - keep_days_or_delete = DELETE You can avoid passing keep_days argument in this case because it will be ignored anyways .. code-block:: python >>> synchronizer = Synchronizer( max_workers=10, policies_manager=policies_manager, dry_run=False, allow_empty_source=True, compare_version_mode=CompareVersionMode.SIZE, compare_threshold=10, # in bytes newer_file_mode=NewerFileSyncMode.REPLACE, keep_days_or_delete=KeepOrDeleteMode.DELETE, ) >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) delete f1.txt delete f1.txt (old version) delete hello.txt (old version) upload f2.txt delete hello.txt (hide marker) As you can see, it deleted f1.txt and it's older versions (no hide this time) and deleted hello.txt also because now we don't want the file anymore. also, we added another file f2.txt which gets uploaded. Now we changed newer_file_mode to SKIP and compare_version_mode to MODTIME. also uploaded a new version of f2.txt to bucket using B2 web. .. code-block:: python >>> synchronizer = Synchronizer( max_workers=10, policies_manager=policies_manager, dry_run=False, allow_empty_source=True, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=10, # in seconds newer_file_mode=NewerFileSyncMode.SKIP, keep_days_or_delete=KeepOrDeleteMode.DELETE, ) >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) As expected, nothing happened, it found a file that was older at source but did not do anything because we skipped. Now we changed newer_file_mode again to REPLACE and also uploaded a new version of f2.txt to bucket using B2 web. .. code-block:: python >>> synchronizer = Synchronizer( max_workers=10, policies_manager=policies_manager, dry_run=False, allow_empty_source=True, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=10, newer_file_mode=NewerFileSyncMode.REPLACE, keep_days_or_delete=KeepOrDeleteMode.DELETE, ) >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) delete f2.txt (old version) upload f2.txt Handling encryption ------------------- The `Synchronizer` object may need `EncryptionSetting` instances to perform downloads and copies. For this reason, the `sync_folder` method accepts an `EncryptionSettingsProvider`, see :ref:`server_side_encryption` for further explanation and :ref:`encryption_provider` for public API. Public API classes ================== .. autoclass:: b2sdk.v2.ScanPoliciesManager() :inherited-members: :special-members: __init__ :members: .. autoclass:: b2sdk.v2.Synchronizer() :inherited-members: :special-members: __init__ :members: .. autoclass:: b2sdk.v2.SyncReport() :inherited-members: :special-members: __init__ :members: .. _encryption_provider: Sync Encryption Settings Providers ================================== .. autoclass:: b2sdk.v2.AbstractSyncEncryptionSettingsProvider() :inherited-members: :members: .. autoclass:: b2sdk.v2.ServerDefaultSyncEncryptionSettingsProvider() :no-members: .. autoclass:: b2sdk.v2.BasicSyncEncryptionSettingsProvider() :special-members: __init__ :no-members: b2-sdk-python-1.17.3/doc/source/api/transfer/000077500000000000000000000000001426424117700206565ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/transfer/emerge/000077500000000000000000000000001426424117700221225ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/transfer/emerge/write_intent.rst000066400000000000000000000002331426424117700253650ustar00rootroot00000000000000Write intent =============================================== .. autoclass:: b2sdk.v2.WriteIntent() :inherited-members: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/transfer/outbound/000077500000000000000000000000001426424117700225155ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/api/transfer/outbound/outbound_source.rst000066400000000000000000000002621426424117700264660ustar00rootroot00000000000000Outbound Transfer Source =============================================== .. autoclass:: b2sdk.v2.OutboundTransferSource() :inherited-members: :special-members: __init__ b2-sdk-python-1.17.3/doc/source/api/utils.rst000066400000000000000000000010041426424117700207170ustar00rootroot00000000000000B2 Utility functions ==================== .. autofunction:: b2sdk.v2.b2_url_encode .. autofunction:: b2sdk.v2.b2_url_decode .. autofunction:: b2sdk.v2.choose_part_ranges .. autofunction:: b2sdk.v2.fix_windows_path_limit .. autofunction:: b2sdk.v2.format_and_scale_fraction .. autofunction:: b2sdk.v2.format_and_scale_number .. autofunction:: b2sdk.v2.hex_sha1_of_stream .. autofunction:: b2sdk.v2.hex_sha1_of_bytes .. autoclass:: b2sdk.v2.TempDir() :inherited-members: :special-members: __enter__, __exit__ b2-sdk-python-1.17.3/doc/source/api_reference.rst000066400000000000000000000035721426424117700216110ustar00rootroot00000000000000.. hint:: Use :doc:`quick_start` to quickly jump to examples ######################## API Reference ######################## Interface types ======================= **b2sdk** API is divided into two parts, :ref:`public ` and :ref:`internal `. Please pay attention to which interface type you use. .. tip:: :ref:`Pinning versions ` properly ensures the stability of your application. .. _api_public: Public API ======================== .. toctree:: api/application_key api/account_info api/cache api/api api/exception api/bucket api/file_lock api/data_classes api/downloaded_file api/enums api/progress api/sync api/utils api/transfer/emerge/write_intent api/transfer/outbound/outbound_source api/encryption/setting api/encryption/types .. _api_internal: Internal API ======================== .. note:: See :ref:`Internal interface ` chapter to learn when and how to safely use the Internal API .. toctree:: api/internal/session api/internal/raw_api api/internal/b2http api/internal/requests api/internal/utils api/internal/cache api/internal/stream/chained api/internal/stream/hashing api/internal/stream/progress api/internal/stream/range api/internal/stream/wrapper api/internal/scan/folder_parser api/internal/scan/folder api/internal/scan/path api/internal/scan/policies api/internal/scan/scan api/internal/sync/action api/internal/sync/exception api/internal/sync/policy api/internal/sync/policy_manager api/internal/sync/sync api/internal/transfer/inbound/downloader/abstract api/internal/transfer/inbound/downloader/parallel api/internal/transfer/inbound/downloader/simple api/internal/transfer/inbound/download_manager api/internal/transfer/outbound/upload_source api/internal/raw_simulator b2-sdk-python-1.17.3/doc/source/api_types.rst000066400000000000000000000156121426424117700210150ustar00rootroot00000000000000######################## About API interfaces ######################## .. _semantic_versioning: ******************* Semantic versioning ******************* **b2sdk** follows `Semantic Versioning `_ policy, so in essence the version number is ``MAJOR.MINOR.PATCH`` (for example ``1.2.3``) and: - we increase `MAJOR` version when we make **incompatible** API changes - we increase `MINOR` version when we add functionality **in a backwards-compatible manner**, and - we increase `PATCH` version when we make backwards-compatible **bug fixes** (unless someone relies on the undocumented behavior of a fixed bug) Therefore when setting up **b2sdk** as a dependency, please make sure to match the version appropriately, for example you could put this in your ``requirements.txt`` to make sure your code is compatible with the ``b2sdk`` version your user will get from pypi:: b2sdk>=1.15.0,<2.0.0 .. _interface_versions: ****************** Interface versions ****************** You might notice that the import structure provided in the documentation looks a little odd: ``from b2sdk.v2 import ...``. The ``.v2`` part is used to keep the interface fluid without risk of breaking applications that use the old signatures. With new versions, **b2sdk** will provide functions with signatures matching the old ones, wrapping the new interface in place of the old one. What this means for a developer using **b2sdk**, is that it will just keep working. We have already deleted some legacy functions when moving from ``.v0`` to ``.v1``, providing equivalent wrappers to reduce the migration effort for applications using pre-1.0 versions of **b2sdk** to fixing imports. It also means that **b2sdk** developers may change the interface in the future and will not need to maintain many branches and backport fixes to keep compatibility of for users of those old branches. .. _interface_version_compatibility: ******************************* Interface version compatibility ******************************* A :term:`numbered interface` will not be exactly identical throughout its lifespan, which should not be a problem for anyone, however just in case, the acceptable differences that the developer must tolerate, are listed below. Exceptions ========== The exception hierarchy may change in a backwards compatible manner and the developer must anticipate it. For example, if ``b2sdk.v2.ExceptionC`` inherits directly from ``b2sdk.v2.ExceptionA``, it may one day inherit from ``b2sdk.v2.ExceptionB``, which in turn inherits from ``b2sdk.v2.ExceptionA``. Normally this is not a problem if you use ``isinstance()`` and ``super()`` properly, but your code should not call the constructor of a parent class by directly naming it or it might skip the middle class of the hierarchy (``ExceptionB`` in this example). Extensions ========== Even in the same interface version, objects/classes/enums can get additional fields and their representations such as ``as_dict()`` or ``__repr__`` (but not ``__str__``) may start to contain those fields. Methods and functions can start accepting new **optional** arguments. New methods can be added to existing classes. Performance =========== Some effort will be put into keeping the performance of the old interfaces, but in rare situations old interfaces may end up with a slightly degraded performance after a new version of the library is released. If performance target is absolutely critical to your application, you can pin your dependencies to the middle version (using ``b2sdk>=X.Y.0,` section. This should be used in 99% of use cases, it's enough to implement anything from a `console tool `_ to a `FUSE filesystem `_. Those modules will generally not change in a backwards-incompatible way between non-major versions. Please see :ref:`interface version compatibility ` chapter for notes on what changes must be expected. .. note:: Replication is currently in a Closed Beta state, where not all B2 accounts have access to the feature. The interface of the beta server API might change and the interface of **b2sdk** around replication may change as well. For the avoidance of doubt, until this message is removed, replication-related functionality of **b2sdk** should be considered as internal interface. .. hint:: If the current version of **b2sdk** is ``4.5.6`` and you only use the *public* interface, put this in your ``requirements.txt`` to be safe:: b2sdk>=4.5.6,<5.0.0 .. note:: ``b2sdk.*._something`` and ``b2sdk.*.*._something``, while having a name beginning with an underscore, are **NOT** considered public interface. .. _internal_interface: ****************** Internal interface ****************** Some rarely used features of B2 cloud are not implemented in **b2sdk**. Tracking usage of transactions and transferred data is a good example - if it is required, additional work would need to be put into a specialized internal interface layer to enable accounting and reporting. **b2sdk** maintainers are :ref:`very supportive ` in case someone wants to contribute an additional feature. Please consider adding it to the sdk, so that more people can use it. This way it will also receive our updates, unlike a private implementation which would not receive any updates unless you apply them manually ( but that's a lot of work and we both know it's not going to happen). In practice, an implementation can be either shared or will quickly become outdated. The license of **b2sdk** is very permissive, but when considering whether to keep your patches private or public, please take into consideration the long-term cost of keeping up with a dynamic open-source project and/or the cost of missing the updates, especially those related to performance and reliability (as those are being actively developed in parallel to documentation). Internal interface modules are listed in :ref:`API Internal ` section. .. note:: It is OK for you to use our internal interface (better that than copying our source files!), however, if you do, please pin your dependencies to **middle** version, as backwards-incompatible changes may be introduced in a non-major version. Furthermore, it would be greatly appreciated if an issue was filed for such situations, so that **b2sdk** interface can be improved in a future version in order to avoid strict version pinning. .. hint:: If the current version of **b2sdk** is ``4.5.6`` and you are using the *internal* interface, put this in your requirements.txt:: b2sdk>=4.5.6,<4.6.0 b2-sdk-python-1.17.3/doc/source/conf.py000066400000000000000000000147461426424117700175740ustar00rootroot00000000000000###################################################################### # # File: doc/source/conf.py # # Copyright 2019, Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # -*- coding: utf-8 -*- # # B2 Python SDK documentation build configuration file, created by # sphinx-quickstart on Fri Oct 20 18:2doc/source/conf.py # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys sys.path.append(os.path.abspath('../..')) from b2sdk.version import VERSION # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.coverage', 'sphinx.ext.graphviz', 'sphinx.ext.autosummary', 'sphinx.ext.todo', #'sphinxcontrib.fulltoc', # 2019-03-29: unfortunately this doesn't work with sphinx_rtd_theme 'sphinxcontrib.plantuml', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = u'b2-sdk-python' copyright = u'2020, Backblaze' author = u'Backblaze' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = VERSION.rsplit('.', 1)[0] # The full version, including alpha/beta/rc tags. release = VERSION # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = os.environ.get('B2_SPHINX_TODO', False) and True # -- Options for HTML output ---------------------------------------------- html_context = { "display_github": True, # Add 'Edit on Github' link instead of 'View page source' "github_user": "Backblaze", "github_repo": project, "github_version": "master", "conf_py_path": "/doc/source/", "source_suffix": source_suffix, } # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { 'prev_next_buttons_location': 'both', 'collapse_navigation': True, } autodoc_default_options = { 'member-order': 'bysource', 'exclude-members': '__weakref__, _abc_cache, _abc_negative_cache, _abc_negative_cache_version, _abc_registry, _abc_impl', 'members': True, 'undoc-members': True, } # yapf: disable # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". #html_static_path = ['_static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. # # This is required for the alabaster theme # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars html_sidebars = { '**': [ 'about.html', 'navigation.html', 'relations.html', # needs 'show_related': True theme option to display 'searchbox.html', 'donate.html', ] } # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = 'B2_Python_SDKdoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'B2_Python_SDK.tex', u'B2\\_Python\\_SDK', u'Backblaze', 'manual'), ] # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [(master_doc, 'b2_python_sdk', u'B2 Python SDK Documentation', [author], 1)] # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ( master_doc, 'B2_Python_SDK', u'B2 Python SDK Documentation', author, 'B2_Python_SDK', 'Backblaze Python SDK', 'Miscellaneous' ), ] # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} b2-sdk-python-1.17.3/doc/source/contributing.rst000066400000000000000000000064261426424117700215320ustar00rootroot00000000000000.. _contributors_guide: ######################################### Contributors Guide ######################################### We encourage outside contributors to perform changes on our codebase. Many such changes have been merged already. In order to make it easier to contribute, core developers of this project: * provide guidance (through the issue reporting system) * provide tool assisted code review (through the Pull Request system) * maintain a set of unit tests * maintain a set of integration tests (run with a production cloud) * maintain development automation tools using `nox `_ that can easily: * format the code using `yapf `_ * runs linters to find subtle/potential issues with maintainability * run the test suite on multiple Python versions using `pytest `_ * maintain Continuous Integration (by using GitHub Actions) that: * runs all sorts of linters * checks if the Python distribution can be built * runs all tests on a matrix of 6 versions of Python (including pypy) and 3 operating systems (Linux, Mac OS X and Windows) * checks if the documentation can be built properly * maintain other Continuous Integration tools (coverage tracker) You'll need to have `nox `_ installed: * ``pip install nox`` With ``nox``, you can run different sessions (default are ``lint`` and ``test``): * ``format`` -> Format the code. * ``lint`` -> Run linters. * ``test`` (``test-3.7``, ``test-3.8``, ``test-3.9``, ``test-3.10``) -> Run test suite. * ``cover`` -> Perform coverage analysis. * ``build`` -> Build the distribution. * ``deploy`` -> Deploy the distribution to the PyPi. * ``doc`` -> Build the documentation. * ``doc_cover`` -> Perform coverage analysis for the documentation. For example:: $ nox -s format nox > Running session format nox > Creating virtual environment (virtualenv) using python3.10 in .nox/format ... $ nox -s format nox > Running session format nox > Re-using existing virtual environment at .nox/format. ... $ nox --no-venv -s format nox > Running session format ... Sessions ``test``, ``unit``, and ``integration`` can run on many Python versions, 3.7-3.10 by default. Sessions other than ``test`` use the last given Python version, 3.10 by default. You can change it:: export NOX_PYTHONS=3.7,3.8 With the above setting, session ``test`` will run on Python 3.7 and 3.8, and all other sessions on Python 3.8. Given Python interpreters should be installed in the operating system or via `pyenv `_. Linting ############# To run all available linters:: $ nox -s lint Testing ############# To run all tests on every available Python version:: $ nox -s test To run all tests on a specific version:: $ nox -s test-3.10 To run just unit tests:: $ nox -s unit-3.10 To run just integration tests:: $ export B2_TEST_APPLICATION_KEY=your_app_key $ export B2_TEST_APPLICATION_KEY_ID=your_app_key_id $ nox -s integration-3.10 Documentation ############# To build the documentation and watch for changes (including the source code):: $ nox -s doc To just build the documentation:: $ nox --non-interactive -s doc b2-sdk-python-1.17.3/doc/source/dot/000077500000000000000000000000001426424117700170475ustar00rootroot00000000000000b2-sdk-python-1.17.3/doc/source/dot/sqlite_account_info_schema.dot000066400000000000000000000116021426424117700251270ustar00rootroot00000000000000 digraph G { label = "generated by sadisplay v0.4.9"; fontname = "Bitstream Vera Sans" fontsize = 8 node [ fontname = "Bitstream Vera Sans" fontsize = 8 shape = "plaintext" ] edge [ fontname = "Bitstream Vera Sans" fontsize = 8 ] account [label=<
account
⚪ account_auth_tokenTEXT
⚪ account_idTEXT
⚪ account_id_or_app_key_idTEXT
⚪ allowedTEXT
⚪ api_urlTEXT
⚪ application_keyTEXT
⚪ download_urlTEXT
⚪ minimum_part_sizeINTEGER
⚪ realmTEXT
>] bucket [label=<
bucket
⚪ bucket_idTEXT
⚪ bucket_nameTEXT
>] bucket_upload_url [label=<
bucket_upload_url
⚪ bucket_idTEXT
⚪ upload_auth_tokenTEXT
⚪ upload_urlTEXT
>] update_done [label=<
update_done
⚪ update_numberINTEGER
>] edge [ arrowhead = empty ] edge [ arrowhead = ediamond arrowtail = open ] } b2-sdk-python-1.17.3/doc/source/glossary.rst000066400000000000000000000042651426424117700206650ustar00rootroot00000000000000######## Glossary ######## .. glossary:: absoluteMinimumPartSize The smallest large file part size, as indicated during authorization process by the server (in 2019 it used to be ``5MB``, but the server can set it dynamincally) account ID An identifier of the B2 account (not login). Looks like this: ``4ba5845d7aaf``. application key ID Since every :term:`account ID` can have multiple access keys associated with it, the keys need to be distinguished from each other. :term:`application key ID` is an identifier of the access key. There are two types of keys: :term:`master application key` and :term:`non-master application key`. application key The secret associated with an :term:`application key ID`, used to authenticate with the server. Looks like this: ``N2Zug0evLcHDlh_L0Z0AJhiGGdY`` or ``0a1bce5ea463a7e4b090ef5bd6bd82b851928ab2c6`` or ``K0014pbwo1zxcIVMnqSNTfWHReU/O3s`` b2sdk version Looks like this: ``v1.0.0`` or ``1.0.0`` and makes version numbers meaningful. See :ref:`Pinning versions ` for more details. b2sdk interface version Looks like this: ``v2`` or ``b2sdk.v2`` and makes maintaining backward compatibility much easier. See :ref:`interface versions ` for more details. master application key This is the first key you have access to, it is available on the B2 web application. This key has all capabilities, access to all :term:`buckets`, and has no file prefix restrictions or expiration. The :term:`application key ID` of the master application key is equal to :term:`account ID`. non-master application key A key which can have restricted capabilities, can only have access to a certain :term:`bucket` or even to just part of it. See ``_ to learn more. Looks like this: ``0014aa9865d6f0000000000b0`` bucket A container that holds files. You can think of buckets as the top-level folders in your B2 Cloud Storage account. There is no limit to the number of files in a bucket, but there is a limit of 100 buckets per account. See ``_ to learn more. b2-sdk-python-1.17.3/doc/source/index.rst000066400000000000000000000041011426424117700201160ustar00rootroot00000000000000.. todolist:: ######################################### Overview ######################################### **b2sdk** is a client library for easy access to all of the capabilities of B2 Cloud Storage. `B2 command-line tool `_ is an example of how it can be used to provide command-line access to the B2 service, but there are many possible applications (including `FUSE filesystems `_, storage backend drivers for backup applications etc). ######################################### Why use b2sdk? ######################################### .. todo:: delete doc/source/b2sdk? .. todo:: describe raw_simulator in detail .. todo:: fix list consistency style in "Why use b2sdk?", add links When building an application which uses B2 cloud, it is possible to implement an independent B2 API client, but using **b2sdk** allows for: - reuse of code that is already written, with hundreds of unit tests - use of **Syncronizer**, a high-performance, parallel rsync-like utility - developer-friendly library :ref:`api version policy ` which guards your program against incompatible changes - `B2 integration checklist `_ is passed automatically - **raw_simulator** makes it easy to mock the B2 cloud for unit testing purposes - reporting progress of operations to an object of your choice - exception hierarchy makes it easy to display informative messages to users - interrupted transfers are automatically continued - **b2sdk** has been developed for 3 years before it version 1.0.0 was released. It's stable and mature. ######################################### Documentation index ######################################### .. toctree:: install tutorial quick_start server_side_encryption advanced glossary api_types api_reference contributing ######################################### Indices and tables ######################################### * :ref:`genindex` * :ref:`modindex` * :ref:`search` b2-sdk-python-1.17.3/doc/source/install.rst000066400000000000000000000011651426424117700204640ustar00rootroot00000000000000######################## Installation Guide ######################## Installing as a dependency ========================== **b2sdk** can simply be added to ``requirements.txt`` (or equivalent such as ``setup.py``, ``.pipfile`` etc). In order to properly set a dependency, see :ref:`versioning chapter ` for details. .. note:: The stability of your application depends on correct :ref:`pinning of versions `. Installing a development version ================================ To install **b2sdk**, checkout the repository and run:: pip install b2sdk in your python environment. b2-sdk-python-1.17.3/doc/source/quick_start.rst000066400000000000000000000452541426424117700213560ustar00rootroot00000000000000.. _quick_start: ######################## Quick Start Guide ######################## *********************** Prepare b2sdk *********************** .. code-block:: python >>> from b2sdk.v2 import * >>> info = InMemoryAccountInfo() >>> b2_api = B2Api(info) >>> application_key_id = '4a5b6c7d8e9f' >>> application_key = '001b8e23c26ff6efb941e237deb182b9599a84bef7' >>> b2_api.authorize_account("production", application_key_id, application_key) .. tip:: Get credentials from B2 website *************** Synchronization *************** .. code-block:: python >>> from b2sdk.v2 import ScanPoliciesManager >>> from b2sdk.v2 import parse_folder >>> from b2sdk.v2 import Synchronizer >>> from b2sdk.v2 import SyncReport >>> import time >>> import sys >>> source = '/home/user1/b2_example' >>> destination = 'b2://example-mybucket-b2' >>> source = parse_folder(source, b2_api) >>> destination = parse_folder(destination, b2_api) >>> policies_manager = ScanPoliciesManager(exclude_all_symlinks=True) >>> synchronizer = Synchronizer( max_workers=10, policies_manager=policies_manager, dry_run=False, allow_empty_source=True, ) >>> no_progress = False >>> encryption_settings_provider = BasicSyncEncryptionSettingsProvider({ 'bucket1': EncryptionSettings(mode=EncryptionMode.SSE_B2), 'bucket2': EncryptionSettings( mode=EncryptionMode.SSE_C, key=EncryptionKey(secret=b'VkYp3s6v9y$B&E)H@McQfTjWmZq4t7w!', id='user-generated-key-id') ), 'bucket3': None, }) >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, encryption_settings_provider=encryption_settings_provider, ) upload some.pdf upload som2.pdf .. tip:: Sync is the preferred way of getting data into and out of B2 cloud, because it can achieve *highest performance* due to parallelization of scanning and data transfer operations. To learn more about sync, see :ref:`sync`. Sync uses an encryption provider. In principle, it's a mapping between file metadata (bucket_name, file_info, etc) and `EncryptionSetting`. The reason for employing such a mapping, rather than a single `EncryptionSetting`, is the fact that users of Sync do not necessarily know up front what files it's going to upload and download. This approach enables using unique keys, or key identifiers, across files. This is covered in greater detail in :ref:`server_side_encryption`. In the example above, Sync will assume `SSE-B2` for all files in `bucket1`, `SSE-C` with the key provided for `bucket2` and rely on bucket default for `bucket3`. Should developers need to provide keys per file (and not per bucket), they need to implement their own :class:`b2sdk.v2.AbstractSyncEncryptionSettingsProvider`. ************** Bucket actions ************** List buckets ============ .. code-block:: python >>> b2_api.list_buckets() [Bucket<346501784642eb3e60980d10,example-mybucket-b2-1,allPublic>] >>> for b in b2_api.list_buckets(): print('%s %-10s %s' % (b.id_, b.type_, b.name)) 346501784642eb3e60980d10 allPublic example-mybucket-b2-1 Create a bucket =============== .. code-block:: python >>> bucket_name = 'example-mybucket-b2-1' # must be unique in B2 (across all accounts!) >>> bucket_type = 'allPublic' # or 'allPrivate' >>> b2_api.create_bucket(bucket_name, bucket_type) Bucket<346501784642eb3e60980d10,example-mybucket-b2-1,allPublic> You can optionally store bucket info, CORS rules and lifecycle rules with the bucket. See :meth:`b2sdk.v2.B2Api.create_bucket`. Delete a bucket =============== .. code-block:: python >>> bucket_name = 'example-mybucket-b2-to-delete' >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> b2_api.delete_bucket(bucket) returns `None` if successful, raises an exception in case of error. Update bucket info ================== .. code-block:: python >>> new_bucket_type = 'allPrivate' >>> bucket_name = 'example-mybucket-b2' >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> new_bucket = bucket.update( bucket_type=new_bucket_type, default_server_side_encryption=EncryptionSetting(mode=EncryptionMode.SSE_B2) ) >>> new_bucket.as_dict() {'accountId': '451862be08d0', 'bucketId': '5485a1682662eb3e60980d10', 'bucketInfo': {}, 'bucketName': 'example-mybucket-b2', 'bucketType': 'allPrivate', 'corsRules': [], 'lifecycleRules': [], 'revision': 3, 'defaultServerSideEncryption': {'isClientAuthorizedToRead': True, 'value': {'algorithm': 'AES256', 'mode': 'SSE-B2'}}}, } For more information see :meth:`b2sdk.v2.Bucket.update`. ************ File actions ************ .. tip:: Sync is the preferred way of getting files into and out of B2 cloud, because it can achieve *highest performance* due to parallelization of scanning and data transfer operations. To learn more about sync, see :ref:`sync`. Use the functions described below only if you *really* need to transfer a single file. Upload file =========== .. code-block:: python >>> local_file_path = '/home/user1/b2_example/new.pdf' >>> b2_file_name = 'dummy_new.pdf' >>> file_info = {'how': 'good-file'} >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> bucket.upload_local_file( local_file=local_file_path, file_name=b2_file_name, file_infos=file_info, ) This will work regardless of the size of the file - ``upload_local_file`` automatically uses large file upload API when necessary. For more information see :meth:`b2sdk.v2.Bucket.upload_local_file`. Upload file encrypted with SSE-C -------------------------------- .. code-block:: python >>> local_file_path = '/home/user1/b2_example/new.pdf' >>> b2_file_name = 'dummy_new.pdf' >>> file_info = {'how': 'good-file'} >>> encryption_setting = EncryptionSetting( mode=EncryptionMode.SSE_C, key=EncryptionKey(secret=b'VkYp3s6v9y$B&E)H@McQfTjWmZq4t7w!', id='user-generated-key-id'), ) >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> bucket.upload_local_file( local_file=local_file_path, file_name=b2_file_name, file_infos=file_info, encryption=encryption_setting, ) Download file ============= .. _download-file-by-id: By id ----- .. code-block:: python >>> from b2sdk.v2 import DoNothingProgressListener >>> local_file_path = '/home/user1/b2_example/new2.pdf' >>> file_id = '4_z5485a1682662eb3e60980d10_f1195145f42952533_d20190403_m130258_c002_v0001111_t0002' >>> progress_listener = DoNothingProgressListener() >>> downloaded_file = b2_api.download_file_by_id(file_id, progress_listener) # only the headers # and the beginning of the file is downloaded at this stage >>> print('File name: ', downloaded_file.download_version.file_name) File name: som2.pdf >>> print('File id: ', downloaded_file.download_version.id_) File id: 4_z5485a1682662eb3e60980d10_f1195145f42952533_d20190403_m130258_c002_v0001111_t0002 >>> print('File size: ', downloaded_file.download_version.size) File size: 1870579 >>> print('Content type:', downloaded_file.download_version.content_type) Content type: application/pdf >>> print('Content sha1:', downloaded_file.download_version.content_sha1) Content sha1: d821849a70922e87c2b0786c0be7266b89d87df0 >>> downloaded_file.save_to(local_file_path) # this downloads the whole file .. _download-file-by-name: By name ------- .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> b2_file_name = 'dummy_new.pdf' >>> local_file_name = '/home/user1/b2_example/new3.pdf' >>> downloaded_file = bucket.download_file_by_name(b2_file_name) >>> downloaded_file.save_to(local_file_path) Downloading encrypted files --------------------------- Both methods (:ref:`download-file-by-name` and :ref:`download-file-by-id`) accept an optional `encryption` argument, similarly to `Upload file`_. This parameter is necessary for downloading files encrypted with `SSE-C`. List files ========== .. code-block:: python >>> bucket_name = 'example-mybucket-b2' >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> for file_version, folder_name in bucket.ls(latest_only=True): >>> print(file_version.file_name, file_version.upload_timestamp, folder_name) f2.txt 1560927489000 None som2.pdf 1554296578000 None some.pdf 1554296579000 None test-folder/.bzEmpty 1561005295000 test-folder/ # Recursive >>> bucket_name = 'example-mybucket-b2' >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> for file_version, folder_name in bucket.ls(latest_only=True, recursive=True): >>> print(file_version.file_name, file_version.upload_timestamp, folder_name) f2.txt 1560927489000 None som2.pdf 1554296578000 None some.pdf 1554296579000 None test-folder/.bzEmpty 1561005295000 test-folder/ test-folder/folder_file.txt 1561005349000 None Note: The files are returned recursively and in order so all files in a folder are printed one after another. The folder_name is returned only for the first file in the folder. .. code-block:: python # Within folder >>> bucket_name = 'example-mybucket-b2' >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> for file_version, folder_name in bucket.ls(folder_to_list='test-folder', latest_only=True): >>> print(file_version.file_name, file_version.upload_timestamp, folder_name) test-folder/.bzEmpty 1561005295000 None test-folder/folder_file.txt 1561005349000 None # list file versions >>> for file_version, folder_name in bucket.ls(latest_only=False): >>> print(file_version.file_name, file_version.upload_timestamp, folder_name) f2.txt 1560927489000 None f2.txt 1560849524000 None som2.pdf 1554296578000 None some.pdf 1554296579000 None For more information see :meth:`b2sdk.v2.Bucket.ls`. Get file metadata ========================= .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> file_version = b2_api.get_file_info(file_id) >>> file_version.as_dict() {'accountId': '451862be08d0', 'action': 'upload', 'bucketId': '5485a1682662eb3e60980d10', 'contentLength': 1870579, 'contentSha1': 'd821849a70922e87c2b0786c0be7266b89d87df0', 'contentType': 'application/pdf', 'fileId': '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', 'fileInfo': {'how': 'good-file', 'sse_c_key_id': 'user-generated-key-id'}, 'fileName': 'dummy_new.pdf', 'uploadTimestamp': 1554361150000, "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-C"}, } Update file lock configuration ============================== .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> file_name = 'dummy.pdf' >>> b2_api.update_file_legal_hold(file_id, file_name, LegalHold.ON) >>> b2_api.update_file_legal_hold( file_id, file_name, FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time() + 100)*1000)) This is low-level file API, for high-level operations see `Direct file operations`_. Copy file ========= .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f118df9ba2c5131e8_d20190619_m065809_c002_v0001126_t0040' >>> new_file_version = bucket.copy(file_id, 'f2_copy.txt') >>> new_file_version.as_dict() {'accountId': '451862be08d0', 'action': 'copy', 'bucketId': '5485a1682662eb3e60980d10', 'contentLength': 124, 'contentSha1': '737637702a0e41dda8b7be79c8db1d369c6eef4a', 'contentType': 'text/plain', 'fileId': '4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', 'fileInfo': {'src_last_modified_millis': '1560848707000'}, 'fileName': 'f2_copy.txt', 'uploadTimestamp': 1561033728000, "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"}} If the ``content length`` is not provided and the file is larger than 5GB, ``copy`` would not succeed and error would be raised. If length is provided, then the file may be copied as a large file. Maximum copy part size can be set by ``max_copy_part_size`` - if not set, it will default to 5GB. If ``max_copy_part_size`` is lower than :term:`absoluteMinimumPartSize`, file would be copied in single request - this may be used to force copy in single request large file that fits in server small file limit. Copying files allows for providing encryption settings for both source and destination files - `SSE-C` encrypted source files cannot be used unless the proper key is provided. If you want to copy just the part of the file, then you can specify the offset and content length: .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f118df9ba2c5131e8_d20190619_m065809_c002_v0001126_t0040' >>> bucket.copy(file_id, 'f2_copy.txt', offset=1024, length=2048) Note that content length is required for offset values other than zero. For more information see :meth:`b2sdk.v2.Bucket.copy`. Delete file =========== .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> file_id_and_name = b2_api.delete_file_version(file_id, 'dummy_new.pdf') >>> file_id_and_name.as_dict() {'action': 'delete', 'fileId': '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', 'fileName': 'dummy_new.pdf'} This is low-level file API, for high-level operations see `Direct file operations`_. Cancel large file uploads ========================= .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> for unfinished_file in bucket.list_unfinished_large_files(): b2_api.cancel_large_file(unfinished_file.file_id, unfinished_file.file_name) ********************** Direct file operations ********************** Methods for manipulating object (file) state mentioned in sections above are low level and useful when users have access to basic information, like file id and name. Many API methods, however, return python objects representing files (:class:`b2sdk.v2.FileVersion` and :class:`b2sdk.v2.DownloadVersion`), that provide high-level access to methods manipulating their state. As a rule, these methods don't change properties of python objects they are called on, but return new objects instead. Obtain file representing objects ================================ :class:`b2sdk.v2.FileVersion` ----------------------------- By id ***** .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> file_version = b2_api.get_file_info(file_id) By listing ********** .. code-block:: python >>> for file_version, folder_name in bucket.ls(latest_only=True, prefix='dir_name'): >>> ... :class:`b2sdk.v2.DownloadVersion` --------------------------------- By id ***** .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> downloaded_file = b2_api.download_file_by_id(file_id) >>> download_version = downloaded_file.download_version By name ******* .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> b2_file_name = 'dummy_new.pdf' >>> downloaded_file = bucket.download_file_by_name(b2_file_name) >>> download_version = downloaded_file.download_version Download (only for :class:`b2sdk.v2.FileVersion`) ================================================= .. code-block:: python >>> file_version.download() >>> # equivalent to >>> b2_api.download_file_by_id(file_version.id_) Delete ====== .. code-block:: python >>> file_version.delete() >>> download_version.delete() >>> # equivalent to >>> b2_api.delete_file_version(file_version.id_, file_version.file_name) >>> b2_api.delete_file_version(download_version.id_, download_version.file_name) Update file lock ================ .. code-block:: python >>> file_version.update_legal_hold(LegalHold.ON) >>> download_version.update_legal_hold(LegalHold.ON) >>> file_version.update_retention( FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time() + 100)*1000)) >>> download_version.update_retention( FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time() + 100)*1000)) >>> # equivalent to >>> b2_api.update_file_legal_hold(file_version.id_, file_version.file_name, LegalHold.ON) >>> b2_api.update_file_legal_hold(download_version.id_, download_version.file_name, LegalHold.ON) >>> b2_api.update_file_legal_hold( file_version.id_, file_version.file_name, FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time() + 100)*1000)) >>> b2_api.update_file_legal_hold( download_version.id_, download_version.file_name, FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time() + 100)*1000)) Usage examples ============== .. code-block:: python >>> for file_version, folder_name in bucket.ls(latest_only=True, prefix='dir_name'): >>> if file_version.mod_time_millis < 1627979193913 and file_version.file_name.endswith('.csv'): >>> if file_version.legal_hold.is_on(): >>> file_version = file_version.update_legal_hold(LegalHold.OFF) >>> file_version.delete() >>> else: >>> file_version.download().save_to(Path('/tmp/dir_name') / file_version.file_name) .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> downloaded_file = b2_api.download_file_by_id(file_id) >>> download_version = downloaded_file.download_version >>> if download_version.content_type == 'video/mp4': >>> downloaded_file.save_to(Path('/tmp/dir_name') / download_version.file_name) >>> if download_version.file_retention != NO_RETENTION_FILE_SETTING: >>> download_version = download_version.update_retention( NO_RETENTION_FILE_SETTING, bypass_governance=True) >>> download_version.delete() b2-sdk-python-1.17.3/doc/source/server_side_encryption.rst000066400000000000000000000063051426424117700236030ustar00rootroot00000000000000.. _server_side_encryption: ######################## Server-Side Encryption ######################## *********************** Cloud *********************** B2 cloud supports `Server-Side Encryption `_. All read and write operations provided by **b2sdk** accept encryption settings as an optional argument. Not supplying this argument means relying on bucket defaults - for **SSE-B2** and for no encryption. In case of **SSE-C**, providing a decryption key is required for successful downloading and copying. *** API *** Methods and classes that require encryption settings all accept an `EncryptionSetting` object, which holds information about present or desired encryption (mode, algorithm, key, key_id). Some, like copy operations, accept two sets of settings (for source and for destination). Sync, however, accepts an `EncryptionSettingsProvider` object, which is an `EncryptionSetting` factory, yielding them based on file metadata. For details, see * :ref:`encryption_setting` * :ref:`encryption_types` * :ref:`encryption_provider` ****************************** High security: use unique keys ****************************** B2 cloud does not promote or discourage either reusing encryption keys or using unique keys for `SEE-C`. In applications requiring enhanced security, using unique key per file is a good strategy. **b2sdk** follows a convention, that makes managing such keys easier: `EncryptionSetting` holds a key identifier, aside from the key itself. This key identifier is saved in the metadata of all files uploaded, created or copied via **b2sdk** methods using `SSE-C`, under `sse_c_key_id` in `fileInfo`. This allows developers to create key managers that map those ids to keys, stored securely in a file or a database. Implementing such managers, and linking them to :class:`b2sdk.v2.AbstractSyncEncryptionSettingsProvider` implementations (necessary for using Sync) is outside of the scope of this library. There is, however, a convention to such managers that authors of this library strongly suggest: if a manager needs to generate a new key-key_id pair for uploading, it's best to commit this data to the underlying storage before commencing the upload. The justification of such approach is: should the key-key_id pair be committed to permanent storage after completing an IO operation, committing could fail after successfully upload the data. This data, however, is now just a random blob, that can never be read, since the key to decrypting it is lost. This approach comes an overhead: to download a file, its `fileInfo` has to be known. This means that fetching metadata is required before downloading. ********************************************* Moderate security: a single key with many ids ********************************************* Should the application's infrastructure require a single key (or a limited set of keys) to be used in a bucket, authors of this library recommend generating a unique key identifier for each file anyway (even though these unique identifiers all point to the same key value). This obfuscates confidential metadata from malicious users, like which files are encrypted with the same key, the total number of different keys, etc. b2-sdk-python-1.17.3/doc/source/tutorial.rst000066400000000000000000000072341426424117700206640ustar00rootroot00000000000000######################################### Tutorial ######################################### *************************** AccountInfo *************************** ``AccountInfo`` object holds information about access keys, tokens, upload urls, as well as a bucket id-name map. It is the first object that you need to create to use **b2sdk**. Using ``AccountInfo``, we'll be able to create a ``B2Api`` object to manage a B2 account. In the tutorial we will use :py:class:`b2sdk.v2.InMemoryAccountInfo`: .. code-block:: python >>> from b2sdk.v2 import InMemoryAccountInfo >>> info = InMemoryAccountInfo() # store credentials, tokens and cache in memory With the ``info`` object in hand, we can now proceed to create a ``B2Api`` object. .. note:: :ref:`AccountInfo` section provides guidance for choosing the correct ``AccountInfo`` class for your application. ********************* Account authorization ********************* .. code-block:: python >>> from b2sdk.v2 import B2Api >>> b2_api = B2Api(info) >>> application_key_id = '4a5b6c7d8e9f' >>> application_key = '001b8e23c26ff6efb941e237deb182b9599a84bef7' >>> b2_api.authorize_account("production", application_key_id, application_key) .. tip:: Get credentials from B2 website To find out more about account authorization, see :meth:`b2sdk.v2.B2Api.authorize_account` *************************** B2Api *************************** *B2Api* allows for account-level operations on a B2 account. Typical B2Api operations ======================== .. currentmodule:: b2sdk.v2.B2Api .. autosummary:: :nosignatures: authorize_account create_bucket delete_bucket list_buckets get_bucket_by_name get_bucket_by_id create_key list_keys delete_key download_file_by_id list_parts cancel_large_file .. code-block:: python >>> b2_api = B2Api(info) to find out more, see :class:`b2sdk.v2.B2Api`. The most practical operation on ``B2Api`` object is :meth:`b2sdk.v2.B2Api.get_bucket_by_name`. *Bucket* allows for operations such as listing a remote bucket or transferring files. *************************** Bucket *************************** Initializing a Bucket ======================== Retrieve an existing Bucket --------------------------- To get a ``Bucket`` object for an existing B2 Bucket: .. code-block:: python >>> b2_api.get_bucket_by_name("example-mybucket-b2-1",) Bucket<346501784642eb3e60980d10,example-mybucket-b2-1,allPublic> Create a new Bucket ------------------------ To create a bucket: .. code-block:: python >>> bucket_name = 'example-mybucket-b2-1' >>> bucket_type = 'allPublic' # or 'allPrivate' >>> b2_api.create_bucket(bucket_name, bucket_type) Bucket<346501784642eb3e60980d10,example-mybucket-b2-1,allPublic> You can optionally store bucket info, CORS rules and lifecycle rules with the bucket. See :meth:`b2sdk.v2.B2Api.create_bucket` for more details. .. note:: Bucket name must be unique in B2 (across all accounts!). Your application should be able to cope with a bucket name collision with another B2 user. Typical Bucket operations ========================= .. currentmodule:: b2sdk.v2.Bucket .. autosummary:: :nosignatures: download_file_by_name upload_local_file upload_bytes ls hide_file delete_file_version get_download_authorization get_download_url update set_type set_info To find out more, see :class:`b2sdk.v2.Bucket`. *************************** Summary *************************** You now know how to use ``AccountInfo``, ``B2Api`` and ``Bucket`` objects. To see examples of some of the methods presented above, visit the :ref:`quick start guide ` section. b2-sdk-python-1.17.3/doc/sqlite_account_info_schema.py000077500000000000000000000020341426424117700227050ustar00rootroot00000000000000###################################################################### # # File: doc/sqlite_account_info_schema.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### """ generates a dot file with SqliteAccountInfo database structure """ import tempfile import operator from sadisplay import describe, render from sqlalchemy import create_engine, MetaData from b2sdk.account_info.sqlite_account_info import SqliteAccountInfo def main(): with tempfile.NamedTemporaryFile() as fp: sqlite_db_name = fp.name SqliteAccountInfo(sqlite_db_name) engine = create_engine('sqlite:///' + sqlite_db_name) meta = MetaData() meta.reflect(bind=engine) tables = set(meta.tables.keys()) desc = describe(map(lambda x: operator.getitem(meta.tables, x), sorted(tables))) print(getattr(render, 'dot')(desc).encode('utf-8')) if __name__ == '__main__': main() b2-sdk-python-1.17.3/noxfile.py000066400000000000000000000155331426424117700162410ustar00rootroot00000000000000###################################################################### # # File: noxfile.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os import subprocess from glob import glob import nox CI = os.environ.get('CI') is not None NOX_PYTHONS = os.environ.get('NOX_PYTHONS') SKIP_COVERAGE = os.environ.get('SKIP_COVERAGE') == 'true' PYTHON_VERSIONS = [ '3.7', '3.8', '3.9', '3.10', '3.11', ] if NOX_PYTHONS is None else NOX_PYTHONS.split(',') PYTHON_DEFAULT_VERSION = PYTHON_VERSIONS[-1] PY_PATHS = ['b2sdk', 'test', 'noxfile.py', 'setup.py'] REQUIREMENTS_FORMAT = ['yapf==0.27'] REQUIREMENTS_LINT = ['yapf==0.27', 'pyflakes==2.4.0', 'pytest==6.2.5', 'liccheck==0.6.2'] REQUIREMENTS_TEST = [ "pytest==6.2.5", "pytest-cov==3.0.0", "pytest-mock==3.6.1", 'pytest-lazy-fixture==0.6.3', 'pyfakefs==4.5.6', 'pytest-xdist==2.5.0', ] REQUIREMENTS_BUILD = ['setuptools>=20.2'] nox.options.reuse_existing_virtualenvs = True nox.options.sessions = [ 'lint', 'test', ] # In CI, use Python interpreter provided by GitHub Actions if CI: nox.options.force_venv_backend = 'none' def install_myself(session, extras=None): """Install from the source.""" arg = '.' if extras: arg += '[%s]' % ','.join(extras) session.install('-e', arg) @nox.session(name='format', python=PYTHON_DEFAULT_VERSION) def format_(session): """Format the code.""" session.install(*REQUIREMENTS_FORMAT) # TODO: incremental mode for yapf session.run('yapf', '--in-place', '--parallel', '--recursive', *PY_PATHS) # TODO: uncomment if we want to use isort and docformatter # session.run('isort', *PY_PATHS) # session.run( # 'docformatter', # '--in-place', # '--recursive', # '--wrap-summaries=100', # '--wrap-descriptions=100', # *PY_PATHS, # ) @nox.session(python=PYTHON_DEFAULT_VERSION) def lint(session): """Run linters.""" install_myself(session) session.install(*REQUIREMENTS_LINT) session.run('yapf', '--diff', '--parallel', '--recursive', *PY_PATHS) # TODO: uncomment if we want to use isort and docformatter # session.run('isort', '--check', *PY_PATHS) # session.run( # 'docformatter', # '--check', # '--recursive', # '--wrap-summaries=100', # '--wrap-descriptions=100', # *PY_PATHS, # ) # TODO: use flake8 instead of pyflakes session.log('pyflakes b2sdk') output = subprocess.run('pyflakes b2sdk', shell=True, check=False, stdout=subprocess.PIPE).stdout.decode().strip() excludes = ['__init__.py', 'exception.py'] output = [l for l in output.splitlines() if all(x not in l for x in excludes)] if output: print('\n'.join(output)) session.error('pyflakes has failed') # session.run('flake8', *PY_PATHS) session.run('pytest', 'test/static') session.run('liccheck', '-s', 'setup.cfg') @nox.session(python=PYTHON_VERSIONS) def unit(session): """Run unit tests.""" install_myself(session) session.install(*REQUIREMENTS_TEST) args = ['--doctest-modules', '-p', 'pyfakefs', '-n', 'auto'] if not SKIP_COVERAGE: args += ['--cov=b2sdk', '--cov-branch', '--cov-report=xml'] # TODO: Use session.parametrize for apiver session.run('pytest', '--api=v3', *args, *session.posargs, 'test/unit') if not SKIP_COVERAGE: args += ['--cov-append'] session.run('pytest', '--api=v2', *args, *session.posargs, 'test/unit') session.run('pytest', '--api=v1', *args, *session.posargs, 'test/unit') session.run('pytest', '--api=v0', *args, *session.posargs, 'test/unit') if not SKIP_COVERAGE and not session.posargs: session.notify('cover') @nox.session(python=PYTHON_VERSIONS) def integration(session): """Run integration tests.""" install_myself(session) session.install(*REQUIREMENTS_TEST) session.run('pytest', '-s', *session.posargs, 'test/integration') @nox.session(python=PYTHON_DEFAULT_VERSION) def cleanup_old_buckets(session): """Remove buckets from previous test runs.""" install_myself(session) session.install(*REQUIREMENTS_TEST) session.run('python', '-m', 'test.integration.cleanup_buckets') @nox.session(python=PYTHON_VERSIONS) def test(session): """Run all tests.""" if session.python: session.notify('unit-{}'.format(session.python)) session.notify('integration-{}'.format(session.python)) else: session.notify('unit') session.notify('integration') @nox.session def cover(session): """Perform coverage analysis.""" session.install('coverage') session.run('coverage', 'report', '--fail-under=75', '--show-missing', '--skip-covered') session.run('coverage', 'erase') @nox.session(python=PYTHON_DEFAULT_VERSION) def build(session): """Build the distribution.""" # TODO: consider using wheel as well session.install(*REQUIREMENTS_BUILD) session.run('python', 'setup.py', 'check', '--metadata', '--strict') session.run('rm', '-rf', 'build', 'dist', 'b2sdk.egg-info', external=True) session.run('python', 'setup.py', 'sdist', *session.posargs) # Set outputs for GitHub Actions if CI: asset_path = glob('dist/*')[0] print('::set-output name=asset_path::', asset_path, sep='') asset_name = os.path.basename(asset_path) print('::set-output name=asset_name::', asset_name, sep='') version = os.environ['GITHUB_REF'].replace('refs/tags/v', '') print('::set-output name=version::', version, sep='') @nox.session(python=PYTHON_DEFAULT_VERSION) def doc(session): """Build the documentation.""" install_myself(session, extras=['doc']) session.cd('doc') sphinx_args = ['-b', 'html', '-T', '-W', 'source', 'build/html'] session.run('rm', '-rf', 'build', external=True) if not session.interactive: session.run('sphinx-build', *sphinx_args) session.notify('doc_cover') else: sphinx_args[-2:-2] = [ '-E', '--open-browser', '--watch', '../b2sdk', '--ignore', '*.pyc', '--ignore', '*~' ] session.run('sphinx-autobuild', *sphinx_args) @nox.session def doc_cover(session): """Perform coverage analysis for the documentation.""" install_myself(session, extras=['doc']) session.cd('doc') sphinx_args = ['-b', 'coverage', '-T', '-W', 'source', 'build/coverage'] report_file = 'build/coverage/python.txt' session.run('sphinx-build', *sphinx_args) session.run('cat', report_file, external=True) with open('build/coverage/python.txt') as fd: # If there is no undocumented files, the report should have only 2 lines (header) if sum(1 for _ in fd) != 2: session.error('sphinx coverage has failed') b2-sdk-python-1.17.3/requirements-doc.txt000066400000000000000000000001161426424117700202410ustar00rootroot00000000000000sadisplay sphinx<6.0 sphinx-autobuild sphinx-rtd-theme sphinxcontrib-plantuml b2-sdk-python-1.17.3/requirements.txt000066400000000000000000000002061426424117700174760ustar00rootroot00000000000000arrow>=1.0.2,<2.0.0 importlib-metadata>=3.3.0; python_version < '3.8' logfury>=1.0.1,<2.0.0 requests>=2.9.1,<3.0.0 tqdm>=4.5.0,<5.0.0 b2-sdk-python-1.17.3/setup.cfg000066400000000000000000000023031426424117700160330ustar00rootroot00000000000000[yapf] based_on_style=facebook COLUMN_LIMIT=100 SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=False SPLIT_PENALTY_AFTER_OPENING_BRACKET=0 [isort] line_length=100 multi_line_output=3 atomic=true include_trailing_comma=true force_grid_wrap=0 lines_after_imports=2 lines_between_types=1 use_parentheses=true [flake8] # TODO: remove E501 once docstrings are formatted ignore=E501,W503,D100,D105,D202 per-file-ignores=__init__.py:F401,F403 max-line-length=100 # TODO: consider lower the complexity to e.g. 10 - it requires some refactoring max-complexity=17 doctests=1 [coverage:run] branch=true [Licenses] # Authorized and unauthorized licenses in LOWER CASE authorized_licenses: bsd new bsd bsd license new bsd license simplified bsd apache apache 2.0 apache software apache software license lgpl gnu lgpl gnu library or lesser general public license (lgpl) lgpl with exceptions or zpl isc license isc license (iscl) mit mit license mozilla public license 2.0 (mpl 2.0) mpl-2.0 psf python software foundation python software foundation license zpl 2.1 unauthorized_licenses: affero agpl gpl v3 gpl v2 gpl b2-sdk-python-1.17.3/setup.py000066400000000000000000000107531426424117700157340ustar00rootroot00000000000000###################################################################### # # File: setup.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### """A setuptools based setup module. See: https://packaging.python.org/en/latest/distributing.html https://github.com/pypa/sampleproject """ import sys # To use a consistent encoding from codecs import open from importlib import import_module # Always prefer setuptools over distutils from setuptools import __version__ as setuptoolsversion from setuptools import find_packages, setup #require at least setuptools 20.2 for PEP 508 conditional dependency support MIN_SETUPTOOLS_VERSION = (20, 2) if tuple(int(x) for x in setuptoolsversion.split('.')[:2]) < MIN_SETUPTOOLS_VERSION: sys.exit( 'setuptools %s.%s or later is required. To fix, try running: pip install "setuptools>=%s.%s"' % (MIN_SETUPTOOLS_VERSION * 2) ) # Get the long description from the README file with open('README.md', encoding='utf-8') as f: long_description = f.read() def read_requirements(extra=None): if extra is not None: file = 'requirements-{}.txt'.format(extra) else: file = 'requirements.txt' with open(file, encoding='utf-8') as f: return f.read().splitlines() setup( name='b2sdk', description='Backblaze B2 SDK', long_description=long_description, long_description_content_type='text/markdown', # The project's main homepage. url='https://github.com/Backblaze/b2-sdk-python', # Author details author='Backblaze, Inc.', author_email='support@backblaze.com', # Choose your license license='MIT', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ # How mature is this project? Common values are # 3 - Alpha # 4 - Beta # 5 - Production/Stable 'Development Status :: 5 - Production/Stable', # Indicate who your project is intended for # ??? What are the right classifiers for a command-line tool? ??? 'Intended Audience :: Developers', 'Topic :: Software Development :: Libraries', # Pick your license as you wish (should match "license" above) 'License :: OSI Approved :: MIT License', # Specify the Python versions you support here. 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', ], # What does your project relate to? keywords='backblaze b2 cloud storage', # You can just specify the packages manually here if your project is # simple. Or you can use find_packages(). packages=find_packages(exclude=['contrib', 'doc', 'test*']), # Alternatively, if you want to distribute just a my_module.py, uncomment # this: # py_modules=["my_module"], # List run-time dependencies here. These will be installed by pip when # your project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=read_requirements(), # List additional groups of dependencies here (e.g. development # dependencies). You can install these using the following syntax, # for example: # $ pip install -e .[dev,test] extras_require={'doc': read_requirements('doc')}, setup_requires=['setuptools_scm<6.0'], use_scm_version=True, # If there are data files included in your packages that need to be # installed, specify them here. package_data={'b2sdk': ['requirements.txt', 'LICENSE']}, # Although 'package_data' is the preferred approach, in some case you may # need to place data files outside of your packages. See: # http://docs.python.org/3.10/distutils/setupscript.html#installing-additional-files # noqa # In this case, 'data_file' will be installed into '/my_data' data_files=[ #('my_data', ['data/data_file']) ], # To provide executable scripts, use entry points in preference to the # "scripts" keyword. Entry points provide cross-platform support and allow # pip to create the appropriate form of executable for the target platform. entry_points={ 'pyinstaller40': ['hook-dirs=b2sdk._pyinstaller:get_hook_dirs'], }, ) b2-sdk-python-1.17.3/test/000077500000000000000000000000001426424117700151735ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/__init__.py000066400000000000000000000004331426424117700173040ustar00rootroot00000000000000###################################################################### # # File: test/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/integration/000077500000000000000000000000001426424117700175165ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/integration/__init__.py000066400000000000000000000013071426424117700216300ustar00rootroot00000000000000###################################################################### # # File: test/integration/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os def get_b2_auth_data(): application_key_id = os.environ.get('B2_TEST_APPLICATION_KEY_ID') if application_key_id is None: raise ValueError('B2_TEST_APPLICATION_KEY_ID is not set.') application_key = os.environ.get('B2_TEST_APPLICATION_KEY') if application_key is None: raise ValueError('B2_TEST_APPLICATION_KEY is not set.') return application_key_id, application_key b2-sdk-python-1.17.3/test/integration/base.py000066400000000000000000000041341426424117700210040ustar00rootroot00000000000000###################################################################### # # File: test/integration/base.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional import http.client import os import random import string import pytest from b2sdk.v2 import current_time_millis from .bucket_cleaner import BucketCleaner from .helpers import GENERAL_BUCKET_NAME_PREFIX, BUCKET_NAME_LENGTH, BUCKET_CREATED_AT_MILLIS, bucket_name_part, authorize class IntegrationTestBase: @pytest.fixture(autouse=True) def set_http_debug(self): if os.environ.get('B2_DEBUG_HTTP'): http.client.HTTPConnection.debuglevel = 1 @pytest.fixture(autouse=True) def save_settings(self, dont_cleanup_old_buckets, b2_auth_data): type(self).dont_cleanup_old_buckets = dont_cleanup_old_buckets type(self).b2_auth_data = b2_auth_data @classmethod def setup_class(cls): cls.this_run_bucket_name_prefix = GENERAL_BUCKET_NAME_PREFIX + bucket_name_part(8) @classmethod def teardown_class(cls): BucketCleaner( cls.dont_cleanup_old_buckets, *cls.b2_auth_data, current_run_prefix=cls.this_run_bucket_name_prefix ).cleanup_buckets() @pytest.fixture(autouse=True) def setup_method(self): self.b2_api, self.info = authorize(self.b2_auth_data) def generate_bucket_name(self): return self.this_run_bucket_name_prefix + bucket_name_part( BUCKET_NAME_LENGTH - len(self.this_run_bucket_name_prefix) ) def write_zeros(self, file, number): line = b'0' * 1000 + b'\n' line_len = len(line) written = 0 while written <= number: file.write(line) written += line_len def create_bucket(self): return self.b2_api.create_bucket( self.generate_bucket_name(), 'allPublic', bucket_info={BUCKET_CREATED_AT_MILLIS: str(current_time_millis())} ) b2-sdk-python-1.17.3/test/integration/bucket_cleaner.py000066400000000000000000000101421426424117700230340ustar00rootroot00000000000000###################################################################### # # File: test/integration/bucket_cleaner.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional from b2sdk.v2 import * from .helpers import GENERAL_BUCKET_NAME_PREFIX, BUCKET_CREATED_AT_MILLIS, authorize ONE_HOUR_MILLIS = 60 * 60 * 1000 class BucketCleaner: def __init__( self, dont_cleanup_old_buckets: bool, b2_application_key_id: str, b2_application_key: str, current_run_prefix: Optional[str] = None ): self.current_run_prefix = current_run_prefix self.dont_cleanup_old_buckets = dont_cleanup_old_buckets self.b2_application_key_id = b2_application_key_id self.b2_application_key = b2_application_key def _should_remove_bucket(self, bucket: Bucket): if self.current_run_prefix and bucket.name.startswith(self.current_run_prefix): return True if self.dont_cleanup_old_buckets: return False if bucket.name.startswith(GENERAL_BUCKET_NAME_PREFIX): if BUCKET_CREATED_AT_MILLIS in bucket.bucket_info: if int(bucket.bucket_info[BUCKET_CREATED_AT_MILLIS] ) < current_time_millis() - ONE_HOUR_MILLIS: return True return False def cleanup_buckets(self): b2_api, _ = authorize((self.b2_application_key_id, self.b2_application_key)) buckets = b2_api.list_buckets() for bucket in buckets: if not self._should_remove_bucket(bucket): print('Skipping bucket removal:', bucket.name) else: print('Trying to remove bucket:', bucket.name) files_leftover = False file_versions = bucket.ls(latest_only=False, recursive=True) for file_version_info, _ in file_versions: if file_version_info.file_retention: if file_version_info.file_retention.mode == RetentionMode.GOVERNANCE: print('Removing retention from file version:', file_version_info.id_) b2_api.update_file_retention( file_version_info.id_, file_version_info.file_name, NO_RETENTION_FILE_SETTING, True ) elif file_version_info.file_retention.mode == RetentionMode.COMPLIANCE: if file_version_info.file_retention.retain_until > current_time_millis(): # yapf: disable print( 'File version: %s cannot be removed due to compliance mode retention' % (file_version_info.id_,) ) files_leftover = True continue elif file_version_info.file_retention.mode == RetentionMode.NONE: pass else: raise ValueError( 'Unknown retention mode: %s' % (file_version_info.file_retention.mode,) ) if file_version_info.legal_hold.is_on(): print('Removing legal hold from file version:', file_version_info.id_) b2_api.update_file_legal_hold( file_version_info.id_, file_version_info.file_name, LegalHold.OFF ) print('Removing file version:', file_version_info.id_) b2_api.delete_file_version(file_version_info.id_, file_version_info.file_name) if files_leftover: print('Unable to remove bucket because some retained files remain') else: print('Removing bucket:', bucket.name) b2_api.delete_bucket(bucket) b2-sdk-python-1.17.3/test/integration/cleanup_buckets.py000077500000000000000000000010341426424117700232400ustar00rootroot00000000000000###################################################################### # # File: test/integration/cleanup_buckets.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from . import get_b2_auth_data from .bucket_cleaner import BucketCleaner from .test_raw_api import cleanup_old_buckets if __name__ == '__main__': cleanup_old_buckets() BucketCleaner(False, *get_b2_auth_data()).cleanup_buckets() b2-sdk-python-1.17.3/test/integration/conftest.py000066400000000000000000000011741426424117700217200ustar00rootroot00000000000000###################################################################### # # File: test/integration/conftest.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest def pytest_addoption(parser): """Add a flag for not cleaning up old buckets""" parser.addoption( '--dont-cleanup-old-buckets', action="store_true", default=False, ) @pytest.fixture def dont_cleanup_old_buckets(request): return request.config.getoption("--dont-cleanup-old-buckets") b2-sdk-python-1.17.3/test/integration/fixtures/000077500000000000000000000000001426424117700213675ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/integration/fixtures/__init__.py000066400000000000000000000007701426424117700235040ustar00rootroot00000000000000###################################################################### # # File: test/integration/fixtures/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os import pytest from .. import get_b2_auth_data @pytest.fixture def b2_auth_data(): try: return get_b2_auth_data() except ValueError as ex: pytest.fail(ex.args[0]) b2-sdk-python-1.17.3/test/integration/helpers.py000066400000000000000000000017101426424117700215310ustar00rootroot00000000000000###################################################################### # # File: test/integration/helpers.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from typing import Optional import os import random import string import pytest from b2sdk.v2 import * GENERAL_BUCKET_NAME_PREFIX = 'sdktst' BUCKET_NAME_CHARS = string.ascii_letters + string.digits + '-' BUCKET_NAME_LENGTH = 50 BUCKET_CREATED_AT_MILLIS = 'created_at_millis' def bucket_name_part(length): return ''.join(random.choice(BUCKET_NAME_CHARS) for _ in range(length)) def authorize(b2_auth_data, api_config=DEFAULT_HTTP_API_CONFIG): info = InMemoryAccountInfo() b2_api = B2Api(info, api_config=api_config) realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') b2_api.authorize_account(realm, *b2_auth_data) return b2_api, info b2-sdk-python-1.17.3/test/integration/test_download.py000066400000000000000000000120021426424117700227310ustar00rootroot00000000000000###################################################################### # # File: test/integration/test_download.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import gzip import io import pathlib from pprint import pprint from typing import Optional from unittest import mock from b2sdk.v2 import * from .fixtures import * # pyflakes: disable from .helpers import authorize from .base import IntegrationTestBase class TestDownload(IntegrationTestBase): def test_large_file(self): bucket = self.create_bucket() with mock.patch.object( self.info, '_recommended_part_size', new=self.info.get_absolute_minimum_part_size() ): download_manager = self.b2_api.services.download_manager with mock.patch.object( download_manager, 'strategies', new=[ ParallelDownloader( min_part_size=self.info.get_absolute_minimum_part_size(), min_chunk_size=download_manager.MIN_CHUNK_SIZE, max_chunk_size=download_manager.MAX_CHUNK_SIZE, thread_pool=download_manager._thread_pool, ) ] ): # let's check that small file downloads fail with these settings zero = bucket.upload_bytes(b'0', 'a_single_zero') with pytest.raises(ValueError) as exc_info: with io.BytesIO() as io_: bucket.download_file_by_name('a_single_zero').save(io_) assert exc_info.value.args == ('no strategy suitable for download was found!',) f = self._file_helper(bucket) if zero._type() != 'large': # if we are here, that's not the production server! assert f.download_version.content_sha1_verified # large files don't have sha1, lets not check def _file_helper( self, bucket, sha1_sum=None, bytes_to_write: Optional[int] = None ) -> DownloadVersion: bytes_to_write = bytes_to_write or int(self.info.get_absolute_minimum_part_size()) * 2 + 1 with TempDir() as temp_dir: temp_dir = pathlib.Path(temp_dir) source_small_file = pathlib.Path(temp_dir) / 'source_small_file' with open(source_small_file, 'wb') as small_file: self.write_zeros(small_file, bytes_to_write) bucket.upload_local_file( source_small_file, 'small_file', sha1_sum=sha1_sum, ) target_small_file = pathlib.Path(temp_dir) / 'target_small_file' f = bucket.download_file_by_name('small_file') f.save_to(target_small_file) assert hex_sha1_of_file(source_small_file) == hex_sha1_of_file(target_small_file) return f def test_small(self): bucket = self.create_bucket() f = self._file_helper(bucket, bytes_to_write=1) assert f.download_version.content_sha1_verified def test_small_unverified(self): bucket = self.create_bucket() f = self._file_helper(bucket, sha1_sum='do_not_verify', bytes_to_write=1) if f.download_version.content_sha1_verified: pprint(f.download_version._get_args_for_clone()) assert not f.download_version.content_sha1_verified def test_gzip(self): bucket = self.create_bucket() with TempDir() as temp_dir: temp_dir = pathlib.Path(temp_dir) source_file = temp_dir / 'compressed_file.gz' downloaded_uncompressed_file = temp_dir / 'downloaded_uncompressed_file' downloaded_compressed_file = temp_dir / 'downloaded_compressed_file' data_to_write = b"I'm about to be compressed and sent to the cloud, yay!\n" * 100 # too short files failed somehow with gzip.open(source_file, 'wb') as gzip_file: gzip_file.write(data_to_write) file_version = bucket.upload_local_file( str(source_file), 'gzipped_file', file_infos={'b2-content-encoding': 'gzip'} ) self.b2_api.download_file_by_id(file_id=file_version.id_).save_to( str(downloaded_compressed_file) ) with open(downloaded_compressed_file, 'rb') as dcf: downloaded_data = dcf.read() with open(source_file, 'rb') as sf: source_data = sf.read() assert downloaded_data == source_data decompressing_api, _ = authorize( self.b2_auth_data, B2HttpApiConfig(decode_content=True) ) decompressing_api.download_file_by_id(file_id=file_version.id_).save_to( str(downloaded_uncompressed_file) ) with open(downloaded_uncompressed_file, 'rb') as duf: assert duf.read() == data_to_write b2-sdk-python-1.17.3/test/integration/test_raw_api.py000066400000000000000000000522561426424117700225630ustar00rootroot00000000000000###################################################################### # # File: test/integration/test_raw_api.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import io import os import random import re import sys import time import traceback import pytest from b2sdk.b2http import B2Http from b2sdk.encryption.setting import EncryptionAlgorithm, EncryptionMode, EncryptionSetting from b2sdk.replication.setting import ReplicationConfiguration, ReplicationRule from b2sdk.replication.types import ReplicationStatus from b2sdk.file_lock import BucketRetentionSetting, NO_RETENTION_FILE_SETTING, RetentionMode, RetentionPeriod from b2sdk.raw_api import ALL_CAPABILITIES, B2RawHTTPApi, REALM_URLS from b2sdk.utils import hex_sha1_of_stream # TODO: rewrite to separate test cases def test_raw_api(dont_cleanup_old_buckets): """ Exercise the code in B2RawHTTPApi by making each call once, just to make sure the parameters are passed in, and the result is passed back. The goal is to be a complete test of B2RawHTTPApi, so the tests for the rest of the code can use the simulator. Prints to stdout if things go wrong. :return: 0 on success, non-zero on failure """ application_key_id = os.environ.get('B2_TEST_APPLICATION_KEY_ID') if application_key_id is None: pytest.fail('B2_TEST_APPLICATION_KEY_ID is not set.') application_key = os.environ.get('B2_TEST_APPLICATION_KEY') if application_key is None: pytest.fail('B2_TEST_APPLICATION_KEY is not set.') print() try: raw_api = B2RawHTTPApi(B2Http()) raw_api_test_helper(raw_api, not dont_cleanup_old_buckets) except Exception: traceback.print_exc(file=sys.stdout) pytest.fail('test_raw_api failed') def authorize_raw_api(raw_api): application_key_id = os.environ.get('B2_TEST_APPLICATION_KEY_ID') if application_key_id is None: print('B2_TEST_APPLICATION_KEY_ID is not set.', file=sys.stderr) sys.exit(1) application_key = os.environ.get('B2_TEST_APPLICATION_KEY') if application_key is None: print('B2_TEST_APPLICATION_KEY is not set.', file=sys.stderr) sys.exit(1) realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') realm_url = REALM_URLS.get(realm, realm) auth_dict = raw_api.authorize_account(realm_url, application_key_id, application_key) return auth_dict def raw_api_test_helper(raw_api, should_cleanup_old_buckets): """ Try each of the calls to the raw api. Raise an exception if anything goes wrong. This uses a Backblaze account that is just for this test. The account uses the free level of service, which should be enough to run this test a reasonable number of times each day. If somebody abuses the account for other things, this test will break and we'll have to do something about it. """ # b2_authorize_account print('b2_authorize_account') auth_dict = authorize_raw_api(raw_api) missing_capabilities = set(ALL_CAPABILITIES) - {'readBuckets', 'listAllBucketNames' } - set(auth_dict['allowed']['capabilities']) assert not missing_capabilities, 'it appears that the raw_api integration test is being run with a non-full key. Missing capabilities: %s' % ( missing_capabilities, ) account_id = auth_dict['accountId'] account_auth_token = auth_dict['authorizationToken'] api_url = auth_dict['apiUrl'] download_url = auth_dict['downloadUrl'] # b2_create_key print('b2_create_key') key_dict = raw_api.create_key( api_url, account_auth_token, account_id, ['readFiles'], 'testKey', None, None, None, ) # b2_list_keys print('b2_list_keys') raw_api.list_keys(api_url, account_auth_token, account_id, 10) # b2_delete_key print('b2_delete_key') raw_api.delete_key(api_url, account_auth_token, key_dict['applicationKeyId']) # b2_create_bucket, with a unique bucket name # Include the account ID in the bucket name to be # sure it doesn't collide with bucket names from # other accounts. print('b2_create_bucket') bucket_name = 'test-raw-api-%s-%d-%d' % ( account_id, int(time.time()), random.randint(1000, 9999) ) # very verbose http debug #import http.client; http.client.HTTPConnection.debuglevel = 1 bucket_dict = raw_api.create_bucket( api_url, account_auth_token, account_id, bucket_name, 'allPublic', is_file_lock_enabled=True, ) bucket_id = bucket_dict['bucketId'] first_bucket_revision = bucket_dict['revision'] ################################# print('b2 / replication') # 1) create source key (read permissions) replication_source_key_dict = raw_api.create_key( api_url, account_auth_token, account_id, [ 'listBuckets', 'listFiles', 'readFiles', 'writeFiles', # Pawel @ 2022-06-21: adding this to make tests pass with a weird server validator ], 'testReplicationSourceKey', None, None, None, ) replication_source_key = replication_source_key_dict['applicationKeyId'] # 2) create source bucket with replication to destination - existing bucket try: # in order to test replication, we need to create a second bucket replication_source_bucket_name = 'test-raw-api-%s-%d-%d' % ( account_id, int(time.time()), random.randint(1000, 9999) ) replication_source_bucket_dict = raw_api.create_bucket( api_url, account_auth_token, account_id, replication_source_bucket_name, 'allPublic', is_file_lock_enabled=True, replication=ReplicationConfiguration( rules=[ ReplicationRule( destination_bucket_id=bucket_id, include_existing_files=True, name='test-rule', ), ], source_key_id=replication_source_key, ), ) assert 'replicationConfiguration' in replication_source_bucket_dict assert replication_source_bucket_dict['replicationConfiguration'] == { 'isClientAuthorizedToRead': True, 'value': { "asReplicationSource": { "replicationRules": [ { "destinationBucketId": bucket_id, "fileNamePrefix": "", "includeExistingFiles": True, "isEnabled": True, "priority": 128, "replicationRuleName": "test-rule" }, ], "sourceApplicationKeyId": replication_source_key, }, "asReplicationDestination": None, }, } # 3) upload test file and check replication status upload_url_dict = raw_api.get_upload_url( api_url, account_auth_token, replication_source_bucket_dict['bucketId'], ) file_contents = b'hello world' file_dict = raw_api.upload_file( upload_url_dict['uploadUrl'], upload_url_dict['authorizationToken'], 'test.txt', len(file_contents), 'text/plain', hex_sha1_of_stream(io.BytesIO(file_contents), len(file_contents)), {'color': 'blue'}, io.BytesIO(file_contents), ) assert ReplicationStatus[file_dict['replicationStatus'].upper() ] == ReplicationStatus.PENDING finally: raw_api.delete_key(api_url, account_auth_token, replication_source_key) # 4) create destination key (write permissions) replication_destination_key_dict = raw_api.create_key( api_url, account_auth_token, account_id, ['listBuckets', 'listFiles', 'writeFiles'], 'testReplicationDestinationKey', None, None, None, ) replication_destination_key = replication_destination_key_dict['applicationKeyId'] # 5) update destination bucket to receive updates try: bucket_dict = raw_api.update_bucket( api_url, account_auth_token, account_id, bucket_id, 'allPublic', replication=ReplicationConfiguration( source_to_destination_key_mapping={ replication_source_key: replication_destination_key, }, ), ) assert bucket_dict['replicationConfiguration'] == { 'isClientAuthorizedToRead': True, 'value': { 'asReplicationDestination': { 'sourceToDestinationKeyMapping': { replication_source_key: replication_destination_key, }, }, 'asReplicationSource': None, }, } finally: raw_api.delete_key( api_url, account_auth_token, replication_destination_key_dict['applicationKeyId'], ) # 6) cleanup: disable replication for destination and remove source bucket_dict = raw_api.update_bucket( api_url, account_auth_token, account_id, bucket_id, 'allPublic', replication=ReplicationConfiguration(), ) assert bucket_dict['replicationConfiguration'] == { 'isClientAuthorizedToRead': True, 'value': None, } _clean_and_delete_bucket( raw_api, api_url, account_auth_token, account_id, replication_source_bucket_dict['bucketId'], ) ################# print('b2_update_bucket') sse_b2_aes = EncryptionSetting( mode=EncryptionMode.SSE_B2, algorithm=EncryptionAlgorithm.AES256, ) sse_none = EncryptionSetting(mode=EncryptionMode.NONE) for encryption_setting, default_retention in [ ( sse_none, BucketRetentionSetting(mode=RetentionMode.GOVERNANCE, period=RetentionPeriod(days=1)) ), (sse_b2_aes, None), (sse_b2_aes, BucketRetentionSetting(RetentionMode.NONE)), ]: bucket_dict = raw_api.update_bucket( api_url, account_auth_token, account_id, bucket_id, 'allPublic', default_server_side_encryption=encryption_setting, default_retention=default_retention, ) # b2_list_buckets print('b2_list_buckets') bucket_list_dict = raw_api.list_buckets(api_url, account_auth_token, account_id) #print(bucket_list_dict) # b2_get_upload_url print('b2_get_upload_url') upload_url_dict = raw_api.get_upload_url(api_url, account_auth_token, bucket_id) upload_url = upload_url_dict['uploadUrl'] upload_auth_token = upload_url_dict['authorizationToken'] # b2_upload_file print('b2_upload_file') file_name = 'test.txt' file_contents = b'hello world' file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), len(file_contents)) file_dict = raw_api.upload_file( upload_url, upload_auth_token, file_name, len(file_contents), 'text/plain', file_sha1, {'color': 'blue'}, io.BytesIO(file_contents), server_side_encryption=sse_b2_aes, ) file_id = file_dict['fileId'] # b2_list_file_versions print('b2_list_file_versions') list_versions_dict = raw_api.list_file_versions(api_url, account_auth_token, bucket_id) assert [file_name] == [f_dict['fileName'] for f_dict in list_versions_dict['files']] # b2_download_file_by_id with auth print('b2_download_file_by_id (auth)') url = raw_api.get_download_url_by_id(download_url, file_id) with raw_api.download_file_from_url(account_auth_token, url) as response: data = next(response.iter_content(chunk_size=len(file_contents))) assert data == file_contents, data # b2_download_file_by_id no auth print('b2_download_file_by_id (no auth)') url = raw_api.get_download_url_by_id(download_url, file_id) with raw_api.download_file_from_url(None, url) as response: data = next(response.iter_content(chunk_size=len(file_contents))) assert data == file_contents, data # b2_download_file_by_name with auth print('b2_download_file_by_name (auth)') url = raw_api.get_download_url_by_name(download_url, bucket_name, file_name) with raw_api.download_file_from_url(account_auth_token, url) as response: data = next(response.iter_content(chunk_size=len(file_contents))) assert data == file_contents, data # b2_download_file_by_name no auth print('b2_download_file_by_name (no auth)') url = raw_api.get_download_url_by_name(download_url, bucket_name, file_name) with raw_api.download_file_from_url(None, url) as response: data = next(response.iter_content(chunk_size=len(file_contents))) assert data == file_contents, data # b2_get_download_authorization print('b2_get_download_authorization') download_auth = raw_api.get_download_authorization( api_url, account_auth_token, bucket_id, file_name[:-2], 12345 ) download_auth_token = download_auth['authorizationToken'] # b2_download_file_by_name with download auth print('b2_download_file_by_name (download auth)') url = raw_api.get_download_url_by_name(download_url, bucket_name, file_name) with raw_api.download_file_from_url(download_auth_token, url) as response: data = next(response.iter_content(chunk_size=len(file_contents))) assert data == file_contents, data # b2_list_file_names print('b2_list_file_names') list_names_dict = raw_api.list_file_names(api_url, account_auth_token, bucket_id) assert [file_name] == [f_dict['fileName'] for f_dict in list_names_dict['files']] # b2_list_file_names (start, count) print('b2_list_file_names (start, count)') list_names_dict = raw_api.list_file_names( api_url, account_auth_token, bucket_id, start_file_name=file_name, max_file_count=5 ) assert [file_name] == [f_dict['fileName'] for f_dict in list_names_dict['files']] # b2_copy_file print('b2_copy_file') copy_file_name = 'test_copy.txt' raw_api.copy_file(api_url, account_auth_token, file_id, copy_file_name) # b2_get_file_info_by_id print('b2_get_file_info_by_id') file_info_dict = raw_api.get_file_info_by_id(api_url, account_auth_token, file_id) assert file_info_dict['fileName'] == file_name # b2_get_file_info_by_name print('b2_get_file_info_by_name (no auth)') info_headers = raw_api.get_file_info_by_name(download_url, None, bucket_name, file_name) assert info_headers['x-bz-file-id'] == file_id # b2_get_file_info_by_name print('b2_get_file_info_by_name (auth)') info_headers = raw_api.get_file_info_by_name( download_url, account_auth_token, bucket_name, file_name ) assert info_headers['x-bz-file-id'] == file_id # b2_get_file_info_by_name print('b2_get_file_info_by_name (download auth)') info_headers = raw_api.get_file_info_by_name( download_url, download_auth_token, bucket_name, file_name ) assert info_headers['x-bz-file-id'] == file_id # b2_hide_file print('b2_hide_file') raw_api.hide_file(api_url, account_auth_token, bucket_id, file_name) # b2_start_large_file print('b2_start_large_file') file_info = {'color': 'red'} large_info = raw_api.start_large_file( api_url, account_auth_token, bucket_id, file_name, 'text/plain', file_info, server_side_encryption=sse_b2_aes, ) large_file_id = large_info['fileId'] # b2_get_upload_part_url print('b2_get_upload_part_url') upload_part_dict = raw_api.get_upload_part_url(api_url, account_auth_token, large_file_id) upload_part_url = upload_part_dict['uploadUrl'] upload_path_auth = upload_part_dict['authorizationToken'] # b2_upload_part print('b2_upload_part') part_contents = b'hello part' part_sha1 = hex_sha1_of_stream(io.BytesIO(part_contents), len(part_contents)) raw_api.upload_part( upload_part_url, upload_path_auth, 1, len(part_contents), part_sha1, io.BytesIO(part_contents) ) # b2_copy_part print('b2_copy_part') raw_api.copy_part(api_url, account_auth_token, file_id, large_file_id, 2, (0, 5)) # b2_list_parts print('b2_list_parts') parts_response = raw_api.list_parts(api_url, account_auth_token, large_file_id, 1, 100) assert [1, 2] == [part['partNumber'] for part in parts_response['parts']] # b2_list_unfinished_large_files unfinished_list = raw_api.list_unfinished_large_files(api_url, account_auth_token, bucket_id) assert [file_name] == [f_dict['fileName'] for f_dict in unfinished_list['files']] assert file_info == unfinished_list['files'][0]['fileInfo'] # b2_finish_large_file print('b2_finish_large_file') try: raw_api.finish_large_file(api_url, account_auth_token, large_file_id, [part_sha1]) raise Exception('finish should have failed') except Exception as e: assert 'large files must have at least 2 parts' in str(e) # TODO: make another attempt to finish but this time successfully # b2_update_bucket print('b2_update_bucket') updated_bucket = raw_api.update_bucket( api_url, account_auth_token, account_id, bucket_id, 'allPrivate', bucket_info={'color': 'blue'}, default_retention=BucketRetentionSetting( mode=RetentionMode.GOVERNANCE, period=RetentionPeriod(days=1) ), ) assert first_bucket_revision < updated_bucket['revision'] # Clean up this test. _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id) if should_cleanup_old_buckets: # Clean up from old tests. Empty and delete any buckets more than an hour old. _cleanup_old_buckets(raw_api, auth_dict, bucket_list_dict) def cleanup_old_buckets(): raw_api = B2RawHTTPApi(B2Http()) auth_dict = authorize_raw_api(raw_api) bucket_list_dict = raw_api.list_buckets( auth_dict['apiUrl'], auth_dict['authorizationToken'], auth_dict['accountId'] ) _cleanup_old_buckets(raw_api, auth_dict, bucket_list_dict) def _cleanup_old_buckets(raw_api, auth_dict, bucket_list_dict): for bucket_dict in bucket_list_dict['buckets']: bucket_id = bucket_dict['bucketId'] bucket_name = bucket_dict['bucketName'] if _should_delete_bucket(bucket_name): print('cleaning up old bucket: ' + bucket_name) _clean_and_delete_bucket( raw_api, auth_dict['apiUrl'], auth_dict['authorizationToken'], auth_dict['accountId'], bucket_id, ) def _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id): # Delete the files. This test never creates more than a few files, # so one call to list_file_versions should get them all. versions_dict = raw_api.list_file_versions(api_url, account_auth_token, bucket_id) for version_dict in versions_dict['files']: file_id = version_dict['fileId'] file_name = version_dict['fileName'] action = version_dict['action'] if action in ['hide', 'upload']: print('b2_delete_file', file_name, action) if action == 'upload' and version_dict[ 'fileRetention'] and version_dict['fileRetention']['value']['mode'] is not None: raw_api.update_file_retention( api_url, account_auth_token, file_id, file_name, NO_RETENTION_FILE_SETTING, bypass_governance=True ) raw_api.delete_file_version(api_url, account_auth_token, file_id, file_name) else: print('b2_cancel_large_file', file_name) raw_api.cancel_large_file(api_url, account_auth_token, file_id) # Delete the bucket print('b2_delete_bucket', bucket_id) raw_api.delete_bucket(api_url, account_auth_token, account_id, bucket_id) def _should_delete_bucket(bucket_name): # Bucket names for this test look like: c7b22d0b0ad7-1460060364-5670 # Other buckets should not be deleted. match = re.match(r'^test-raw-api-[a-f0-9]+-([0-9]+)-([0-9]+)', bucket_name) if match is None: return False # Is it more than an hour old? bucket_time = int(match.group(1)) now = time.time() return bucket_time + 3600 <= now b2-sdk-python-1.17.3/test/static/000077500000000000000000000000001426424117700164625ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/static/__init__.py000066400000000000000000000004421426424117700205730ustar00rootroot00000000000000###################################################################### # # File: test/static/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/static/test_licenses.py000066400000000000000000000016171426424117700217050ustar00rootroot00000000000000###################################################################### # # File: test/static/test_licenses.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from glob import glob from itertools import islice import pytest def test_files_headers(): for file in glob('**/*.py', recursive=True): with open(file) as fd: file = file.replace( '\\', '/' ) # glob('**/*.py') on Windows returns "b2\bucket.py" (wrong slash) head = ''.join(islice(fd, 9)) if 'All Rights Reserved' not in head: pytest.fail('Missing "All Rights Reserved" in the header in: {}'.format(file)) if file not in head: pytest.fail('Wrong file name in the header in: {}'.format(file)) b2-sdk-python-1.17.3/test/unit/000077500000000000000000000000001426424117700161525ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/__init__.py000066400000000000000000000004401426424117700202610ustar00rootroot00000000000000###################################################################### # # File: test/unit/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/account_info/000077500000000000000000000000001426424117700206215ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/account_info/__init__.py000066400000000000000000000004551426424117700227360ustar00rootroot00000000000000###################################################################### # # File: test/unit/account_info/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/account_info/fixtures.py000066400000000000000000000061041426424117700230450ustar00rootroot00000000000000###################################################################### # # File: test/unit/account_info/fixtures.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from apiver_deps import InMemoryAccountInfo, SqliteAccountInfo @pytest.fixture def account_info_default_data_schema_0(): return dict( account_id='account_id', auth_token='account_auth', api_url='https://api000.backblazeb2.xyz:8180', download_url='https://f000.backblazeb2.xyz:8180', minimum_part_size=100, application_key='app_key', realm='dev', ) @pytest.fixture def account_info_default_data(account_info_default_data_schema_0, apiver): if apiver in ['v0', 'v1']: return dict( allowed=None, application_key_id='application_key_id', s3_api_url='https://s3.us-west-000.backblazeb2.xyz:8180', **account_info_default_data_schema_0, ) return dict( allowed=None, application_key_id='application_key_id', s3_api_url='https://s3.us-west-000.backblazeb2.xyz:8180', account_id='account_id', auth_token='account_auth', api_url='https://api000.backblazeb2.xyz:8180', download_url='https://f000.backblazeb2.xyz:8180', recommended_part_size=100, absolute_minimum_part_size=50, application_key='app_key', realm='dev', ) @pytest.fixture(scope='session') def in_memory_account_info_factory(): def get_account_info(): return InMemoryAccountInfo() return get_account_info @pytest.fixture def in_memory_account_info(in_memory_account_info_factory): return in_memory_account_info_factory() @pytest.fixture def sqlite_account_info_factory(tmpdir): def get_account_info(file_name=None, schema_0=False): if file_name is None: file_name = str(tmpdir.join('b2_account_info')) if schema_0: last_upgrade_to_run = 0 else: last_upgrade_to_run = None return SqliteAccountInfo(file_name, last_upgrade_to_run) return get_account_info @pytest.fixture def sqlite_account_info(sqlite_account_info_factory): return sqlite_account_info_factory() @pytest.fixture( params=[ pytest.lazy_fixture('in_memory_account_info_factory'), pytest.lazy_fixture('sqlite_account_info_factory'), ] ) def account_info_factory(request): return request.param @pytest.fixture( params=[ pytest.lazy_fixture('in_memory_account_info'), pytest.lazy_fixture('sqlite_account_info'), ] ) def account_info(request): return request.param @pytest.fixture def fake_account_info(mocker): account_info = mocker.MagicMock(name='FakeAccountInfo', spec=InMemoryAccountInfo) account_info.REALM_URLS = { 'dev': 'http://api.backblazeb2.xyz:8180', } account_info.is_same_account.return_value = True account_info.is_same_key.return_value = True return account_info b2-sdk-python-1.17.3/test/unit/account_info/test_account_info.py000066400000000000000000000441451426424117700247110ustar00rootroot00000000000000###################################################################### # # File: test/unit/account_info/test_account_info.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from abc import ABCMeta, abstractmethod import json import unittest.mock as mock import os import platform import shutil import stat import tempfile import pytest from apiver_deps import ( ALL_CAPABILITIES, AbstractAccountInfo, InMemoryAccountInfo, UploadUrlPool, SqliteAccountInfo, TempDir, B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR, ) from apiver_deps_exception import CorruptAccountInfo, MissingAccountData from .fixtures import * class WindowsSafeTempDir(TempDir): def __exit__(self, exc_type, exc_val, exc_tb): try: super().__exit__(exc_type, exc_val, exc_tb) except OSError: pass class TestAccountInfo: @pytest.fixture(autouse=True) def setup(self, account_info_factory, account_info_default_data): self.account_info_factory = account_info_factory self.account_info_default_data = account_info_default_data @pytest.mark.parametrize( 'application_key_id,realm,expected', ( ('application_key_id', 'dev', True), ('application_key_id', 'test', False), ('different_application_key_id', 'dev', False), ('different_application_key_id', 'test', False), ), ) def test_is_same_key(self, application_key_id, realm, expected): account_info = self.account_info_factory() account_info.set_auth_data(**self.account_info_default_data) assert account_info.is_same_key(application_key_id, realm) is expected @pytest.mark.parametrize( 'account_id,application_key_id,expected', ( ('account_id', 'account_id', True), ('account_id', 'ACCOUNT_ID', False), ('account_id', '123account_id0000000000', True), ('account_id', '234account_id0000000000', True), ('account_id', '123account_id000000000', False), ('account_id', '123account_id0000000001', False), ('account_id', '123account_id00000000000', False), ), ) def test_is_master_key(self, account_id, application_key_id, expected): account_info = self.account_info_factory() account_data = self.account_info_default_data.copy() account_data['account_id'] = account_id account_data['application_key_id'] = application_key_id account_info.set_auth_data(**account_data) assert account_info.is_master_key() is expected, (account_id, application_key_id, expected) @pytest.mark.parametrize( 'account_id,realm,expected', ( ('account_id', 'dev', True), ('account_id', 'test', False), ('different_account_id', 'dev', False), ('different_account_id', 'test', False), ), ) def test_is_same_account(self, account_id, realm, expected): account_info = self.account_info_factory() account_info.set_auth_data(**self.account_info_default_data) assert account_info.is_same_account(account_id, realm) is expected @pytest.mark.parametrize( 's3_api_url', ('https://s3.us-east-123.backblazeb2.com', 'https://s3.us-west-321.backblazeb2.com') ) def test_s3_api_url(self, s3_api_url): account_info = self.account_info_factory() account_info_default_data = { **self.account_info_default_data, 's3_api_url': s3_api_url, } account_info.set_auth_data(**account_info_default_data) assert s3_api_url == account_info.get_s3_api_url() def test_getting_all_capabilities(self): account_info = self.account_info_factory() assert account_info.all_capabilities() == ALL_CAPABILITIES class TestUploadUrlPool: @pytest.fixture(autouse=True) def setUp(self): self.pool = UploadUrlPool() def test_take_empty(self): assert (None, None) == self.pool.take('a') def test_put_and_take(self): self.pool.put('a', 'url_a1', 'auth_token_a1') self.pool.put('a', 'url_a2', 'auth_token_a2') self.pool.put('b', 'url_b1', 'auth_token_b1') assert ('url_a2', 'auth_token_a2') == self.pool.take('a') assert ('url_a1', 'auth_token_a1') == self.pool.take('a') assert (None, None) == self.pool.take('a') assert ('url_b1', 'auth_token_b1') == self.pool.take('b') assert (None, None) == self.pool.take('b') def test_clear(self): self.pool.put('a', 'url_a1', 'auth_token_a1') self.pool.clear_for_key('a') self.pool.put('b', 'url_b1', 'auth_token_b1') assert (None, None) == self.pool.take('a') assert ('url_b1', 'auth_token_b1') == self.pool.take('b') assert (None, None) == self.pool.take('b') class AccountInfoBase(metaclass=ABCMeta): # it is a mixin to avoid running the tests directly (without inheritance) PERSISTENCE = NotImplemented # subclass should override this @abstractmethod def _make_info(self): """ returns a new object of AccountInfo class which should be tested """ def test_clear(self, account_info_default_data, apiver): account_info = self._make_info() account_info.set_auth_data(**account_info_default_data) account_info.clear() with pytest.raises(MissingAccountData): account_info.get_account_id() with pytest.raises(MissingAccountData): account_info.get_account_auth_token() with pytest.raises(MissingAccountData): account_info.get_api_url() with pytest.raises(MissingAccountData): account_info.get_application_key() with pytest.raises(MissingAccountData): account_info.get_download_url() with pytest.raises(MissingAccountData): account_info.get_realm() with pytest.raises(MissingAccountData): account_info.get_application_key_id() assert not account_info.is_same_key('key_id', 'realm') if apiver in ['v0', 'v1']: with pytest.raises(MissingAccountData): account_info.get_minimum_part_size() else: with pytest.raises(MissingAccountData): account_info.get_recommended_part_size() with pytest.raises(MissingAccountData): account_info.get_absolute_minimum_part_size() def test_set_auth_data_compatibility(self, account_info_default_data): account_info = self._make_info() # The original set_auth_data account_info.set_auth_data(**account_info_default_data) actual = account_info.get_allowed() assert AbstractAccountInfo.DEFAULT_ALLOWED == actual, 'default allowed' # allowed was added later allowed = dict( bucketId=None, bucketName=None, capabilities=['readFiles'], namePrefix=None, ) account_info.set_auth_data(**{ **account_info_default_data, 'allowed': allowed, }) assert allowed == account_info.get_allowed() def test_clear_bucket_upload_data(self): account_info = self._make_info() account_info.put_bucket_upload_url('bucket-0', 'http://bucket-0', 'bucket-0_auth') account_info.clear_bucket_upload_data('bucket-0') assert (None, None) == account_info.take_bucket_upload_url('bucket-0') def test_large_file_upload_urls(self): account_info = self._make_info() account_info.put_large_file_upload_url('file_0', 'http://file_0', 'auth_0') assert ('http://file_0', 'auth_0') == account_info.take_large_file_upload_url('file_0') assert (None, None) == account_info.take_large_file_upload_url('file_0') def test_clear_large_file_upload_urls(self): account_info = self._make_info() account_info.put_large_file_upload_url('file_0', 'http://file_0', 'auth_0') account_info.clear_large_file_upload_urls('file_0') assert (None, None) == account_info.take_large_file_upload_url('file_0') def test_bucket(self): account_info = self._make_info() bucket = mock.MagicMock() bucket.name = 'my-bucket' bucket.id_ = 'bucket-0' assert account_info.get_bucket_id_or_none_from_bucket_name('my-bucket') is None assert account_info.get_bucket_name_or_none_from_bucket_id('bucket-0') is None account_info.save_bucket(bucket) assert 'bucket-0' == account_info.get_bucket_id_or_none_from_bucket_name('my-bucket') assert 'my-bucket' == account_info.get_bucket_name_or_none_from_bucket_id('bucket-0') if self.PERSISTENCE: assert 'bucket-0' == self._make_info( ).get_bucket_id_or_none_from_bucket_name('my-bucket') assert 'my-bucket' == self._make_info( ).get_bucket_name_or_none_from_bucket_id('bucket-0') account_info.remove_bucket_name('my-bucket') assert account_info.get_bucket_id_or_none_from_bucket_name('my-bucket') is None assert account_info.get_bucket_name_or_none_from_bucket_id('bucket-0') is None if self.PERSISTENCE: assert self._make_info().get_bucket_id_or_none_from_bucket_name('my-bucket') is None assert self._make_info().get_bucket_name_or_none_from_bucket_id('bucket-0') is None def test_refresh_bucket(self): account_info = self._make_info() assert account_info.get_bucket_id_or_none_from_bucket_name('my-bucket') is None assert account_info.get_bucket_name_or_none_from_bucket_id('a') is None bucket_names = {'a': 'bucket-0', 'b': 'bucket-1'} account_info.refresh_entire_bucket_name_cache(bucket_names.items()) assert 'bucket-0' == account_info.get_bucket_id_or_none_from_bucket_name('a') assert 'a' == account_info.get_bucket_name_or_none_from_bucket_id('bucket-0') if self.PERSISTENCE: assert 'bucket-0' == self._make_info().get_bucket_id_or_none_from_bucket_name('a') assert 'a' == self._make_info().get_bucket_name_or_none_from_bucket_id('bucket-0') @pytest.mark.apiver(to_ver=1) def test_account_info_up_to_v1(self): account_info = self._make_info() account_info.set_auth_data( 'account_id', 'account_auth', 'https://api.backblazeb2.com', 'download_url', 100, 'app_key', 'realm', application_key_id='key_id' ) object_instances = [account_info] if self.PERSISTENCE: object_instances.append(self._make_info()) for info2 in object_instances: assert 'account_id' == info2.get_account_id() assert 'account_auth' == info2.get_account_auth_token() assert 'https://api.backblazeb2.com' == info2.get_api_url() assert 'app_key' == info2.get_application_key() assert 'key_id' == info2.get_application_key_id() assert 'realm' == info2.get_realm() assert 100 == info2.get_minimum_part_size() assert info2.is_same_key('key_id', 'realm') assert not info2.is_same_key('key_id', 'another_realm') assert not info2.is_same_key('another_key_id', 'realm') assert not info2.is_same_key('another_key_id', 'another_realm') @pytest.mark.apiver(from_ver=2) def test_account_info_v2(self): account_info = self._make_info() account_info.set_auth_data( account_id='account_id', auth_token='account_auth', api_url='https://api.backblazeb2.com', download_url='download_url', recommended_part_size=100, absolute_minimum_part_size=50, application_key='app_key', realm='realm', s3_api_url='s3_api_url', allowed=None, application_key_id='key_id', ) object_instances = [account_info] if self.PERSISTENCE: object_instances.append(self._make_info()) for info2 in object_instances: assert 'account_id' == info2.get_account_id() assert 'account_auth' == info2.get_account_auth_token() assert 'https://api.backblazeb2.com' == info2.get_api_url() assert 'app_key' == info2.get_application_key() assert 'key_id' == info2.get_application_key_id() assert 'realm' == info2.get_realm() assert 100 == info2.get_recommended_part_size() assert 50 == info2.get_absolute_minimum_part_size() assert info2.is_same_key('key_id', 'realm') assert not info2.is_same_key('key_id', 'another_realm') assert not info2.is_same_key('another_key_id', 'realm') assert not info2.is_same_key('another_key_id', 'another_realm') class TestInMemoryAccountInfo(AccountInfoBase): PERSISTENCE = False def _make_info(self): return InMemoryAccountInfo() class TestSqliteAccountInfo(AccountInfoBase): PERSISTENCE = True @pytest.fixture(autouse=True) def setUp(self, request): self.db_path = tempfile.NamedTemporaryFile( prefix='tmp_b2_tests_%s__' % (request.node.name,), delete=True ).name try: os.unlink(self.db_path) except OSError: pass self.test_home = tempfile.mkdtemp() yield for cleanup_method in [ lambda: os.unlink(self.db_path), lambda: shutil.rmtree(self.test_home) ]: try: cleanup_method() except OSError: pass @pytest.mark.skipif( platform.system() == 'Windows', reason='different permission system on Windows', ) def test_permissions(self): """ Test that a new database won't be readable by just any user """ s = SqliteAccountInfo(file_name=self.db_path,) mode = os.stat(self.db_path).st_mode assert stat.filemode(mode) == '-rw-------' def test_corrupted(self): """ Test that a corrupted file will be replaced with a blank file. """ with open(self.db_path, 'wb') as f: f.write(b'not a valid database') with pytest.raises(CorruptAccountInfo): self._make_info() @pytest.mark.skipif( platform.system() == 'Windows', reason='it fails to upgrade on Windows, not worth to fix it anymore' ) def test_convert_from_json(self): """ Tests converting from a JSON account info file, which is what version 0.5.2 of the command-line tool used. """ data = dict( account_auth_token='auth_token', account_id='account_id', api_url='api_url', application_key='application_key', download_url='download_url', minimum_part_size=5000, realm='production' ) with open(self.db_path, 'wb') as f: f.write(json.dumps(data).encode('utf-8')) account_info = self._make_info() assert 'auth_token' == account_info.get_account_auth_token() def _make_info(self): return self._make_sqlite_account_info() def _make_sqlite_account_info(self, env=None, last_upgrade_to_run=None): """ Returns a new SqliteAccountInfo that has just read the data from the file. :param dict env: Override Environment variables. """ # Override HOME to ensure hermetic tests with mock.patch('os.environ', env or {'HOME': self.test_home}): return SqliteAccountInfo( file_name=self.db_path if not env else None, last_upgrade_to_run=last_upgrade_to_run, ) def test_uses_default(self): account_info = self._make_sqlite_account_info( env={ 'HOME': self.test_home, 'USERPROFILE': self.test_home, } ) actual_path = os.path.abspath(account_info.filename) assert os.path.join(self.test_home, '.b2_account_info') == actual_path def test_uses_xdg_config_home(self, apiver): with WindowsSafeTempDir() as d: account_info = self._make_sqlite_account_info( env={ 'HOME': self.test_home, 'USERPROFILE': self.test_home, XDG_CONFIG_HOME_ENV_VAR: d, } ) if apiver in ['v0', 'v1']: expected_path = os.path.abspath(os.path.join(self.test_home, '.b2_account_info')) else: assert os.path.exists(os.path.join(d, 'b2')) expected_path = os.path.abspath(os.path.join(d, 'b2', 'account_info')) actual_path = os.path.abspath(account_info.filename) assert expected_path == actual_path def test_uses_existing_file_and_ignores_xdg(self): with WindowsSafeTempDir() as d: default_db_file_location = os.path.join(self.test_home, '.b2_account_info') open(default_db_file_location, 'a').close() account_info = self._make_sqlite_account_info( env={ 'HOME': self.test_home, 'USERPROFILE': self.test_home, XDG_CONFIG_HOME_ENV_VAR: d, } ) actual_path = os.path.abspath(account_info.filename) assert default_db_file_location == actual_path assert not os.path.exists(os.path.join(d, 'b2')) def test_account_info_env_var_overrides_xdg_config_home(self): with WindowsSafeTempDir() as d: account_info = self._make_sqlite_account_info( env={ 'HOME': self.test_home, 'USERPROFILE': self.test_home, XDG_CONFIG_HOME_ENV_VAR: d, B2_ACCOUNT_INFO_ENV_VAR: os.path.join(d, 'b2_account_info'), } ) expected_path = os.path.abspath(os.path.join(d, 'b2_account_info')) actual_path = os.path.abspath(account_info.filename) assert expected_path == actual_path b2-sdk-python-1.17.3/test/unit/account_info/test_sqlite_account_info.py000066400000000000000000000134321426424117700262650ustar00rootroot00000000000000###################################################################### # # File: test/unit/account_info/test_sqlite_account_info.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os import unittest.mock as mock import pytest from apiver_deps import ( B2_ACCOUNT_INFO_DEFAULT_FILE, B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR, AbstractAccountInfo, SqliteAccountInfo, ) from .fixtures import * class TestDatabseMigrations: @pytest.fixture(autouse=True) def setup(self, sqlite_account_info_factory, account_info_default_data_schema_0): self.sqlite_account_info_factory = sqlite_account_info_factory self.account_info_default_data = account_info_default_data_schema_0 def test_upgrade_1_default_allowed(self): """The 'allowed' field should be the default for upgraded databases.""" old_account_info = self.sqlite_account_info_factory(schema_0=True) old_account_info.set_auth_data_with_schema_0_for_test(**self.account_info_default_data) new_account_info = self.sqlite_account_info_factory(file_name=old_account_info.filename) assert AbstractAccountInfo.DEFAULT_ALLOWED == new_account_info.get_allowed() def test_upgrade_2_default_app_key(self): """The 'application_key_id' field should default to the account ID.""" old_account_info = self.sqlite_account_info_factory(schema_0=True) old_account_info.set_auth_data_with_schema_0_for_test(**self.account_info_default_data) new_account_info = self.sqlite_account_info_factory(file_name=old_account_info.filename) assert 'account_id' == new_account_info.get_application_key_id() def test_upgrade_3_default_s3_api_url(self): """The 's3_api_url' field should be set.""" old_account_info = self.sqlite_account_info_factory(schema_0=True) old_account_info.set_auth_data_with_schema_0_for_test(**self.account_info_default_data) new_account_info = self.sqlite_account_info_factory(file_name=old_account_info.filename) assert '' == new_account_info.get_s3_api_url() def test_migrate_to_4(self): old_account_info = self.sqlite_account_info_factory(schema_0=True) old_account_info.set_auth_data_with_schema_0_for_test(**self.account_info_default_data) new_account_info = self.sqlite_account_info_factory(file_name=old_account_info.filename) with new_account_info._get_connection() as conn: sizes = conn.execute( "SELECT recommended_part_size, absolute_minimum_part_size from account" ).fetchone() assert (100, 5000000) == sizes class TestSqliteAccountProfileFileLocation: @pytest.fixture(autouse=True) def setup(self, monkeypatch, fs): monkeypatch.delenv(B2_ACCOUNT_INFO_ENV_VAR, raising=False) monkeypatch.delenv(XDG_CONFIG_HOME_ENV_VAR, raising=False) def test_invalid_profile_name(self): with pytest.raises(ValueError): SqliteAccountInfo._get_user_account_info_path(profile='&@(*$') def test_profile_and_file_name_conflict(self): with pytest.raises(ValueError): SqliteAccountInfo._get_user_account_info_path(file_name='foo', profile='bar') def test_profile_and_env_var_conflict(self, monkeypatch): monkeypatch.setenv(B2_ACCOUNT_INFO_ENV_VAR, 'foo') with pytest.raises(ValueError): SqliteAccountInfo._get_user_account_info_path(profile='bar') def test_profile_and_xdg_config_env_var(self, monkeypatch): monkeypatch.setenv(XDG_CONFIG_HOME_ENV_VAR, os.path.join('~', 'custom')) account_info_path = SqliteAccountInfo._get_user_account_info_path(profile='secondary') assert account_info_path == os.path.expanduser( os.path.join('~', 'custom', 'b2', 'db-secondary.sqlite') ) def test_profile(self): account_info_path = SqliteAccountInfo._get_user_account_info_path(profile='foo') assert account_info_path == os.path.expanduser(os.path.join('~', '.b2db-foo.sqlite')) def test_file_name(self): account_info_path = SqliteAccountInfo._get_user_account_info_path( file_name=os.path.join('~', 'foo') ) assert account_info_path == os.path.expanduser(os.path.join('~', 'foo')) def test_env_var(self, monkeypatch): monkeypatch.setenv(B2_ACCOUNT_INFO_ENV_VAR, os.path.join('~', 'foo')) account_info_path = SqliteAccountInfo._get_user_account_info_path() assert account_info_path == os.path.expanduser(os.path.join('~', 'foo')) def test_default_file_if_exists(self, monkeypatch): # ensure that XDG_CONFIG_HOME_ENV_VAR doesn't matter if default file exists monkeypatch.setenv(XDG_CONFIG_HOME_ENV_VAR, 'some') account_file_path = os.path.expanduser(B2_ACCOUNT_INFO_DEFAULT_FILE) parent_dir = os.path.abspath(os.path.join(account_file_path, os.pardir)) os.makedirs(parent_dir, exist_ok=True) with open(account_file_path, 'w') as account_file: account_file.write('') account_info_path = SqliteAccountInfo._get_user_account_info_path() assert account_info_path == os.path.expanduser(B2_ACCOUNT_INFO_DEFAULT_FILE) def test_xdg_config_env_var(self, monkeypatch): monkeypatch.setenv(XDG_CONFIG_HOME_ENV_VAR, os.path.join('~', 'custom')) account_info_path = SqliteAccountInfo._get_user_account_info_path() assert account_info_path == os.path.expanduser( os.path.join('~', 'custom', 'b2', 'account_info') ) def test_default_file(self): account_info_path = SqliteAccountInfo._get_user_account_info_path() assert account_info_path == os.path.expanduser(B2_ACCOUNT_INFO_DEFAULT_FILE) b2-sdk-python-1.17.3/test/unit/api/000077500000000000000000000000001426424117700167235ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/api/__init__.py000066400000000000000000000004441426424117700210360ustar00rootroot00000000000000###################################################################### # # File: test/unit/api/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/api/test_api.py000066400000000000000000000475121426424117700211160ustar00rootroot00000000000000###################################################################### # # File: test/unit/api/test_api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import time import pytest from contextlib import suppress from unittest import mock from ..test_base import create_key import apiver_deps from apiver_deps import B2Api from apiver_deps import B2HttpApiConfig from apiver_deps import B2Http from apiver_deps import DummyCache from apiver_deps import EncryptionAlgorithm from apiver_deps import EncryptionMode from apiver_deps import EncryptionSetting from apiver_deps import FileIdAndName from apiver_deps import FileRetentionSetting from apiver_deps import InMemoryAccountInfo from apiver_deps import LegalHold from apiver_deps import RawSimulator from apiver_deps import RetentionMode from apiver_deps import NO_RETENTION_FILE_SETTING from apiver_deps import ApplicationKey, FullApplicationKey from apiver_deps_exception import RestrictedBucket, InvalidArgument if apiver_deps.V <= 1: from apiver_deps import FileVersionInfo as VFileVersion else: from apiver_deps import FileVersion as VFileVersion class TestApi: @pytest.fixture(autouse=True) def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = DummyCache() self.api = B2Api( self.account_info, self.cache, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator) ) self.raw_api = self.api.session.raw_api (self.application_key_id, self.master_key) = self.raw_api.create_account() def test_get_file_info(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') created_file = bucket.upload_bytes(b'hello world', 'file') result = self.api.get_file_info(created_file.id_) if apiver_deps.V <= 1: self.maxDiff = None with suppress(KeyError): del result['replicationStatus'] assert result == { 'accountId': 'account-0', 'action': 'upload', 'bucketId': 'bucket_0', 'contentLength': 11, 'contentSha1': '2aae6c35c94fcfb415dbe95f408b9ce91ee846ed', 'contentType': 'b2/x-auto', 'fileId': '9999', 'fileInfo': {}, 'fileName': 'file', 'fileRetention': { 'isClientAuthorizedToRead': True, 'value': { 'mode': None } }, 'legalHold': { 'isClientAuthorizedToRead': True, 'value': None }, 'serverSideEncryption': { 'mode': 'none' }, 'uploadTimestamp': 5000 } else: assert isinstance(result, VFileVersion) assert result == created_file @pytest.mark.parametrize( 'expected_delete_bucket_output', [ pytest.param(None, marks=pytest.mark.apiver(from_ver=1)), pytest.param( { 'accountId': 'account-0', 'bucketName': 'bucket2', 'bucketId': 'bucket_1', 'bucketType': 'allPrivate', 'bucketInfo': {}, 'corsRules': [], 'lifecycleRules': [], 'options': set(), 'revision': 1, 'defaultServerSideEncryption': { 'isClientAuthorizedToRead': True, 'value': { 'mode': 'none' }, }, 'fileLockConfiguration': { 'isClientAuthorizedToRead': True, 'value': { 'defaultRetention': { 'mode': None, 'period': None }, 'isFileLockEnabled': None } }, }, marks=pytest.mark.apiver(to_ver=0) ), ], ) def test_list_buckets(self, expected_delete_bucket_output): self._authorize_account() self.api.create_bucket('bucket1', 'allPrivate') bucket2 = self.api.create_bucket('bucket2', 'allPrivate') delete_output = self.api.delete_bucket(bucket2) if expected_delete_bucket_output is not None: with suppress(KeyError): del delete_output['replicationConfiguration'] assert delete_output == expected_delete_bucket_output self.api.create_bucket('bucket3', 'allPrivate') assert [b.name for b in self.api.list_buckets()] == ['bucket1', 'bucket3'] def test_list_buckets_with_name(self): self._authorize_account() self.api.create_bucket('bucket1', 'allPrivate') self.api.create_bucket('bucket2', 'allPrivate') assert [b.name for b in self.api.list_buckets(bucket_name='bucket1')] == ['bucket1'] def test_buckets_with_encryption(self): self._authorize_account() sse_b2_aes = EncryptionSetting( mode=EncryptionMode.SSE_B2, algorithm=EncryptionAlgorithm.AES256, ) no_encryption = EncryptionSetting(mode=EncryptionMode.NONE,) unknown_encryption = EncryptionSetting(mode=EncryptionMode.UNKNOWN,) b1 = self.api.create_bucket( 'bucket1', 'allPrivate', default_server_side_encryption=sse_b2_aes, ) self._verify_if_bucket_is_encrypted(b1, should_be_encrypted=True) b2 = self.api.create_bucket('bucket2', 'allPrivate') self._verify_if_bucket_is_encrypted(b2, should_be_encrypted=False) # uses list_buckets self._check_if_bucket_is_encrypted('bucket1', should_be_encrypted=True) self._check_if_bucket_is_encrypted('bucket2', should_be_encrypted=False) # update to set encryption on b2 b2.update(default_server_side_encryption=sse_b2_aes) self._check_if_bucket_is_encrypted('bucket1', should_be_encrypted=True) self._check_if_bucket_is_encrypted('bucket2', should_be_encrypted=True) # update to unset encryption again b2.update(default_server_side_encryption=no_encryption) self._check_if_bucket_is_encrypted('bucket1', should_be_encrypted=True) self._check_if_bucket_is_encrypted('bucket2', should_be_encrypted=False) # now check it with no readBucketEncryption permission to see that it's unknown key = create_key(self.api, ['listBuckets'], 'key1') self.api.authorize_account('production', key.id_, key.application_key) buckets = { b.name: b for b in self.api.list_buckets() # scan again with new key } assert buckets['bucket1'].default_server_side_encryption == unknown_encryption assert buckets['bucket2'].default_server_side_encryption == unknown_encryption def _check_if_bucket_is_encrypted(self, bucket_name, should_be_encrypted): buckets = {b.name: b for b in self.api.list_buckets()} bucket = buckets[bucket_name] return self._verify_if_bucket_is_encrypted(bucket, should_be_encrypted) def _verify_if_bucket_is_encrypted(self, bucket, should_be_encrypted): sse_b2_aes = EncryptionSetting( mode=EncryptionMode.SSE_B2, algorithm=EncryptionAlgorithm.AES256, ) no_encryption = EncryptionSetting(mode=EncryptionMode.NONE,) if not should_be_encrypted: assert bucket.default_server_side_encryption == no_encryption else: assert bucket.default_server_side_encryption == sse_b2_aes assert bucket.default_server_side_encryption.mode == EncryptionMode.SSE_B2 assert bucket.default_server_side_encryption.algorithm == EncryptionAlgorithm.AES256 def test_list_buckets_with_id(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') self.api.create_bucket('bucket2', 'allPrivate') assert [b.name for b in self.api.list_buckets(bucket_id=bucket.id_)] == ['bucket1'] def test_reauthorize_with_app_key(self): # authorize and create a key self._authorize_account() key = create_key(self.api, ['listBuckets'], 'key1') # authorize with the key self.api.authorize_account('production', key.id_, key.application_key) # expire the auth token we just got self.raw_api.expire_auth_token(self.account_info.get_account_auth_token()) # listing buckets should work, after it re-authorizes self.api.list_buckets() def test_list_buckets_with_restriction(self): self._authorize_account() bucket1 = self.api.create_bucket('bucket1', 'allPrivate') self.api.create_bucket('bucket2', 'allPrivate') key = create_key(self.api, ['listBuckets'], 'key1', bucket_id=bucket1.id_) self.api.authorize_account('production', key.id_, key.application_key) assert [b.name for b in self.api.list_buckets(bucket_name=bucket1.name)] == ['bucket1'] def test_get_bucket_by_name_with_bucket_restriction(self): self._authorize_account() bucket1 = self.api.create_bucket('bucket1', 'allPrivate') key = create_key(self.api, ['listBuckets'], 'key1', bucket_id=bucket1.id_) self.api.authorize_account('production', key.id_, key.application_key) assert self.api.get_bucket_by_name('bucket1').id_ == bucket1.id_ def test_list_buckets_with_restriction_and_wrong_name(self): self._authorize_account() bucket1 = self.api.create_bucket('bucket1', 'allPrivate') bucket2 = self.api.create_bucket('bucket2', 'allPrivate') key = create_key(self.api, ['listBuckets'], 'key1', bucket_id=bucket1.id_) self.api.authorize_account('production', key.id_, key.application_key) with pytest.raises(RestrictedBucket) as excinfo: self.api.list_buckets(bucket_name=bucket2.name) assert str(excinfo.value) == 'Application key is restricted to bucket: bucket1' def test_list_buckets_with_restriction_and_no_name(self): self._authorize_account() bucket1 = self.api.create_bucket('bucket1', 'allPrivate') self.api.create_bucket('bucket2', 'allPrivate') key = create_key(self.api, ['listBuckets'], 'key1', bucket_id=bucket1.id_) self.api.authorize_account('production', key.id_, key.application_key) with pytest.raises(RestrictedBucket) as excinfo: self.api.list_buckets() assert str(excinfo.value) == 'Application key is restricted to bucket: bucket1' def test_list_buckets_with_restriction_and_wrong_id(self): self._authorize_account() bucket1 = self.api.create_bucket('bucket1', 'allPrivate') self.api.create_bucket('bucket2', 'allPrivate') key = create_key(self.api, ['listBuckets'], 'key1', bucket_id=bucket1.id_) self.api.authorize_account('production', key.id_, key.application_key) with pytest.raises(RestrictedBucket) as excinfo: self.api.list_buckets(bucket_id='not the one bound to the key') assert str(excinfo.value) == 'Application key is restricted to bucket: %s' % (bucket1.id_,) def _authorize_account(self): self.api.authorize_account('production', self.application_key_id, self.master_key) def test_update_file_retention(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate', is_file_lock_enabled=True) created_file = bucket.upload_bytes(b'hello world', 'file') assert created_file.file_retention == NO_RETENTION_FILE_SETTING new_retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 100) read_file_retention = self.api.update_file_retention( created_file.id_, created_file.file_name, new_retention ) assert new_retention == read_file_retention if apiver_deps.V <= 1: file_version = bucket.get_file_info_by_id(created_file.id_) else: file_version = self.api.get_file_info(created_file.id_) assert new_retention == file_version.file_retention def test_update_legal_hold(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate', is_file_lock_enabled=True) created_file = bucket.upload_bytes(b'hello world', 'file') assert created_file.legal_hold == LegalHold.UNSET new_legal_hold = LegalHold.ON read_legal_hold = self.api.update_file_legal_hold( created_file.id_, created_file.file_name, new_legal_hold ) assert new_legal_hold == read_legal_hold if apiver_deps.V <= 1: file_version = bucket.get_file_info_by_id(created_file.id_) else: file_version = self.api.get_file_info(created_file.id_) assert new_legal_hold == file_version.legal_hold @pytest.mark.apiver(from_ver=2) def test_cancel_large_file_v2(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') unfinished_large_file = self.api.services.large_file.start_large_file( bucket.id_, 'a_large_file' ) cancel_result = self.api.cancel_large_file(unfinished_large_file.file_id) assert cancel_result == FileIdAndName('9999', 'a_large_file') @pytest.mark.apiver(to_ver=1) def test_cancel_large_file_v1(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') unfinished_large_file = self.api.services.large_file.start_large_file( bucket.id_, 'a_large_file' ) cancel_result = self.api.cancel_large_file(unfinished_large_file.file_id) assert cancel_result == VFileVersion( id_='9999', file_name='a_large_file', size=0, content_type='unknown', content_sha1='none', file_info={}, upload_timestamp=0, action='cancel', api=self.api, ) @pytest.mark.apiver(to_ver=1) def test_provide_raw_api_v1(self): from apiver_deps import B2RawApi # test for legacy name old_style_api = B2Api(raw_api=B2RawApi(B2Http(user_agent_append='test append'))) new_style_api = B2Api(api_config=B2HttpApiConfig(user_agent_append='test append')) assert old_style_api.session.raw_api.b2_http.user_agent == new_style_api.session.raw_api.b2_http.user_agent with pytest.raises(InvalidArgument): B2Api( raw_api=B2RawApi(B2Http(user_agent_append='test append')), api_config=B2HttpApiConfig(user_agent_append='test append'), ) @pytest.mark.apiver(to_ver=1) def test_create_and_delete_key_v1(self): self._authorize_account() create_result = self.api.create_key(['readFiles'], 'testkey') assert create_result == { 'accountId': self.account_info.get_account_id(), 'applicationKey': 'appKey0', 'applicationKeyId': 'appKeyId0', 'capabilities': ['readFiles'], 'keyName': 'testkey', } delete_result = self.api.delete_key(create_result['applicationKeyId']) self.assertDeleteAndCreateResult(create_result, delete_result) create_result = self.api.create_key(['readFiles'], 'testkey') delete_result = self.api.delete_key_by_id(create_result['applicationKeyId']) self.assertDeleteAndCreateResult(create_result, delete_result.as_dict()) @pytest.mark.apiver(from_ver=2) def test_create_and_delete_key_v2(self): self._authorize_account() bucket = self.api.create_bucket('bucket', 'allPrivate') now = time.time() create_result = self.api.create_key( ['readFiles'], 'testkey', valid_duration_seconds=100, bucket_id=bucket.id_, name_prefix='name', ) assert isinstance(create_result, FullApplicationKey) assert create_result.key_name == 'testkey' assert create_result.capabilities == ['readFiles'] assert create_result.account_id == self.account_info.get_account_id() assert (now + 100 - 10) * 1000 < create_result.expiration_timestamp_millis < (now + 100 + 10) * 1000 assert create_result.bucket_id == bucket.id_ assert create_result.name_prefix == 'name' # assert create_result.options == ... TODO delete_result = self.api.delete_key(create_result) self.assertDeleteAndCreateResult(create_result, delete_result) create_result = self.api.create_key( ['readFiles'], 'testkey', valid_duration_seconds=100, bucket_id=bucket.id_, name_prefix='name', ) delete_result = self.api.delete_key_by_id(create_result.id_) self.assertDeleteAndCreateResult(create_result, delete_result) def assertDeleteAndCreateResult(self, create_result, delete_result): if apiver_deps.V <= 1: create_result.pop('applicationKey') assert delete_result == create_result else: assert isinstance(delete_result, ApplicationKey) assert delete_result.key_name == create_result.key_name assert delete_result.capabilities == create_result.capabilities assert delete_result.account_id == create_result.account_id assert delete_result.expiration_timestamp_millis == create_result.expiration_timestamp_millis assert delete_result.bucket_id == create_result.bucket_id assert delete_result.name_prefix == create_result.name_prefix @pytest.mark.apiver(to_ver=1) def test_list_keys_v1(self): self._authorize_account() for i in range(20): self.api.create_key(['readFiles'], 'testkey%s' % (i,)) with mock.patch.object(self.api, 'DEFAULT_LIST_KEY_COUNT', 10): response = self.api.list_keys() assert response['nextApplicationKeyId'] == 'appKeyId18' assert response['keys'] == [ { 'accountId': 'account-0', 'applicationKeyId': 'appKeyId%s' % (ind,), 'bucketId': None, 'capabilities': ['readFiles'], 'expirationTimestamp': None, 'keyName': 'testkey%s' % (ind,), 'namePrefix': None, } for ind in [ 0, 1, 10, 11, 12, 13, 14, 15, 16, 17, ] ] @pytest.mark.apiver(from_ver=2) def test_list_keys_v2(self): self._authorize_account() for i in range(20): self.api.create_key(['readFiles'], 'testkey%s' % (i,)) with mock.patch.object(self.api, 'DEFAULT_LIST_KEY_COUNT', 10): keys = list(self.api.list_keys()) assert [key.id_ for key in keys] == [ 'appKeyId0', 'appKeyId1', 'appKeyId10', 'appKeyId11', 'appKeyId12', 'appKeyId13', 'appKeyId14', 'appKeyId15', 'appKeyId16', 'appKeyId17', 'appKeyId18', 'appKeyId19', 'appKeyId2', 'appKeyId3', 'appKeyId4', 'appKeyId5', 'appKeyId6', 'appKeyId7', 'appKeyId8', 'appKeyId9', ] assert isinstance(keys[0], ApplicationKey) b2-sdk-python-1.17.3/test/unit/b2http/000077500000000000000000000000001426424117700173555ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/b2http/__init__.py000066400000000000000000000004471426424117700214730ustar00rootroot00000000000000###################################################################### # # File: test/unit/b2http/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/b2http/test_b2http.py000066400000000000000000000265571426424117700222100ustar00rootroot00000000000000###################################################################### # # File: test/unit/b2http/test_b2http.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import datetime import requests import socket from ..test_base import TestBase import apiver_deps from apiver_deps_exception import BadDateFormat, BadJson, BrokenPipe, B2ConnectionError, ClockSkew, ConnectionReset, ServiceError, UnknownError, UnknownHost, TooManyRequests from apiver_deps import USER_AGENT from apiver_deps import B2Http from apiver_deps import B2HttpApiConfig from apiver_deps import ClockSkewHook from unittest.mock import call, MagicMock, patch class TestTranslateErrors(TestBase): def test_ok(self): response = MagicMock() response.status_code = 200 actual = B2Http._translate_errors(lambda: response) self.assertIs(response, actual) def test_partial_content(self): response = MagicMock() response.status_code = 206 actual = B2Http._translate_errors(lambda: response) self.assertIs(response, actual) def test_b2_error(self): response = MagicMock() response.status_code = 503 response.content = b'{"status": 503, "code": "server_busy", "message": "busy"}' with self.assertRaises(ServiceError): B2Http._translate_errors(lambda: response) def test_broken_pipe(self): def fcn(): raise requests.ConnectionError( requests.packages.urllib3.exceptions.ProtocolError( "dummy", socket.error(20, 'Broken pipe') ) ) with self.assertRaises(BrokenPipe): B2Http._translate_errors(fcn) def test_unknown_host(self): def fcn(): raise requests.ConnectionError( requests.packages.urllib3.exceptions.MaxRetryError( 'AAA nodename nor servname provided, or not known AAA', 'http://example.com' ) ) with self.assertRaises(UnknownHost): B2Http._translate_errors(fcn) def test_connection_error(self): def fcn(): raise requests.ConnectionError('a message') with self.assertRaises(B2ConnectionError): B2Http._translate_errors(fcn) def test_connection_reset(self): class SysCallError(Exception): pass def fcn(): raise SysCallError('(104, ECONNRESET)') with self.assertRaises(ConnectionReset): B2Http._translate_errors(fcn) def test_unknown_error(self): def fcn(): raise Exception('a message') with self.assertRaises(UnknownError): B2Http._translate_errors(fcn) def test_too_many_requests(self): response = MagicMock() response.status_code = 429 response.headers = {'retry-after': 1} response.content = b'{"status": 429, "code": "Too Many requests", "message": "retry after some time"}' with self.assertRaises(TooManyRequests): B2Http._translate_errors(lambda: response) class TestTranslateAndRetry(TestBase): def setUp(self): self.response = MagicMock() self.response.status_code = 200 def test_works_first_try(self): fcn = MagicMock() fcn.side_effect = [self.response] self.assertIs(self.response, B2Http._translate_and_retry(fcn, 3)) def test_non_retryable(self): with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [BadJson('a'), self.response] with self.assertRaises(BadJson): B2Http._translate_and_retry(fcn, 3) self.assertEqual([], mock_time.mock_calls) def test_works_second_try(self): with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [ServiceError('a'), self.response] self.assertIs(self.response, B2Http._translate_and_retry(fcn, 3)) self.assertEqual([call(1.0)], mock_time.mock_calls) def test_never_works(self): with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [ ServiceError('a'), ServiceError('a'), ServiceError('a'), self.response ] with self.assertRaises(ServiceError): B2Http._translate_and_retry(fcn, 3) self.assertEqual([call(1.0), call(1.5)], mock_time.mock_calls) def test_too_many_requests_works_after_sleep(self): with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [TooManyRequests(retry_after_seconds=2), self.response] self.assertIs(self.response, B2Http._translate_and_retry(fcn, 3)) self.assertEqual([call(2)], mock_time.mock_calls) def test_too_many_requests_failed_after_sleep(self): with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [ TooManyRequests(retry_after_seconds=2), TooManyRequests(retry_after_seconds=5), ] with self.assertRaises(TooManyRequests): B2Http._translate_and_retry(fcn, 2) self.assertEqual([call(2)], mock_time.mock_calls) def test_too_many_requests_retry_header_combination_one(self): # If the first response didn't have a header, second one has, and third one doesn't have, what should happen? with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [ TooManyRequests(retry_after_seconds=2), TooManyRequests(), TooManyRequests(retry_after_seconds=2), self.response, ] self.assertIs(self.response, B2Http._translate_and_retry(fcn, 4)) self.assertEqual([call(2), call(1.5), call(2)], mock_time.mock_calls) def test_too_many_requests_retry_header_combination_two(self): # If the first response had header, and the second did not, but the third has header again, what should happen? with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [ TooManyRequests(), TooManyRequests(retry_after_seconds=5), TooManyRequests(), self.response, ] self.assertIs(self.response, B2Http._translate_and_retry(fcn, 4)) self.assertEqual([call(1.0), call(5), call(2.25)], mock_time.mock_calls) class TestB2Http(TestBase): URL = 'http://example.com' UA_APPEND = None HEADERS = dict(my_header='my_value') EXPECTED_HEADERS = {'my_header': 'my_value', 'User-Agent': USER_AGENT} PARAMS = dict(fileSize=100) PARAMS_JSON_BYTES = b'{"fileSize": 100}' def setUp(self): self.session = MagicMock() self.response = MagicMock() requests = MagicMock() requests.Session.return_value = self.session if apiver_deps.V <= 1: self.b2_http = B2Http( requests, install_clock_skew_hook=False, user_agent_append=self.UA_APPEND ) else: self.b2_http = B2Http( B2HttpApiConfig( requests.Session, install_clock_skew_hook=False, user_agent_append=self.UA_APPEND ) ) def test_post_json_return_json(self): self.session.post.return_value = self.response self.response.status_code = 200 self.response.content = b'{"color": "blue"}' response_dict = self.b2_http.post_json_return_json(self.URL, self.HEADERS, self.PARAMS) self.assertEqual({'color': 'blue'}, response_dict) (pos_args, kw_args) = self.session.post.call_args self.assertEqual(self.URL, pos_args[0]) self.assertEqual(self.EXPECTED_HEADERS, kw_args['headers']) actual_data = kw_args['data'] actual_data.seek(0) self.assertEqual(self.PARAMS_JSON_BYTES, actual_data.read()) def test_callback(self): callback = MagicMock() callback.pre_request = MagicMock() callback.post_request = MagicMock() self.b2_http.add_callback(callback) self.session.post.return_value = self.response self.response.status_code = 200 self.response.content = b'{"color": "blue"}' self.b2_http.post_json_return_json(self.URL, self.HEADERS, self.PARAMS) callback.pre_request.assert_called_with('POST', 'http://example.com', self.EXPECTED_HEADERS) callback.post_request.assert_called_with( 'POST', 'http://example.com', self.EXPECTED_HEADERS, self.response ) def test_get_content(self): self.session.get.return_value = self.response self.response.status_code = 200 with self.b2_http.get_content(self.URL, self.HEADERS) as r: self.assertIs(self.response, r) self.session.get.assert_called_with( self.URL, headers=self.EXPECTED_HEADERS, stream=True, timeout=B2Http.TIMEOUT ) self.response.close.assert_called_with() def test_head_content(self): self.session.head.return_value = self.response self.response.status_code = 200 self.response.headers = {"color": "blue"} response = self.b2_http.head_content(self.URL, self.HEADERS) self.assertEqual({'color': 'blue'}, response.headers) (pos_args, kw_args) = self.session.head.call_args self.assertEqual(self.URL, pos_args[0]) self.assertEqual(self.EXPECTED_HEADERS, kw_args['headers']) class TestB2HttpUserAgentAppend(TestB2Http): UA_APPEND = 'ua_extra_string' EXPECTED_HEADERS = { **TestB2Http.EXPECTED_HEADERS, 'User-Agent': '%s %s' % (USER_AGENT, UA_APPEND) } class TestClockSkewHook(TestBase): def test_bad_format(self): response = MagicMock() response.headers = {'Date': 'bad format'} with self.assertRaises(BadDateFormat): ClockSkewHook().post_request('POST', 'http://example.com', {}, response) def test_bad_month(self): response = MagicMock() response.headers = {'Date': 'Fri, 16 XXX 2016 20:52:30 GMT'} with self.assertRaises(BadDateFormat): ClockSkewHook().post_request('POST', 'http://example.com', {}, response) def test_no_skew(self): now = datetime.datetime.utcnow() now_str = now.strftime('%a, %d %b %Y %H:%M:%S GMT') response = MagicMock() response.headers = {'Date': now_str} ClockSkewHook().post_request('POST', 'http://example.com', {}, response) def test_positive_skew(self): now = datetime.datetime.utcnow() + datetime.timedelta(minutes=11) now_str = now.strftime('%a, %d %b %Y %H:%M:%S GMT') response = MagicMock() response.headers = {'Date': now_str} with self.assertRaises(ClockSkew): ClockSkewHook().post_request('POST', 'http://example.com', {}, response) def test_negative_skew(self): now = datetime.datetime.utcnow() + datetime.timedelta(minutes=-11) now_str = now.strftime('%a, %d %b %Y %H:%M:%S GMT') response = MagicMock() response.headers = {'Date': now_str} with self.assertRaises(ClockSkew): ClockSkewHook().post_request('POST', 'http://example.com', {}, response) b2-sdk-python-1.17.3/test/unit/bucket/000077500000000000000000000000001426424117700174275ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/bucket/__init__.py000066400000000000000000000004471426424117700215450ustar00rootroot00000000000000###################################################################### # # File: test/unit/bucket/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/bucket/test_bucket.py000066400000000000000000002451761426424117700223340ustar00rootroot00000000000000###################################################################### # # File: test/unit/bucket/test_bucket.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import io from contextlib import suppress from io import BytesIO import os import platform import unittest.mock as mock import pytest from ..test_base import TestBase, create_key import apiver_deps from apiver_deps_exception import ( AlreadyFailed, B2Error, B2RequestTimeoutDuringUpload, BucketIdNotFound, InvalidAuthToken, InvalidMetadataDirective, InvalidRange, InvalidUploadSource, MaxRetriesExceeded, UnsatisfiableRange, FileSha1Mismatch, SSECKeyError, ) if apiver_deps.V <= 1: from apiver_deps import DownloadDestBytes, PreSeekedDownloadDest from apiver_deps import FileVersionInfo as VFileVersionInfo else: DownloadDestBytes, PreSeekedDownloadDest = None, None # these classes are not present, thus not needed, in v2 from apiver_deps import FileVersion as VFileVersionInfo from apiver_deps import B2Api from apiver_deps import B2HttpApiConfig from apiver_deps import Bucket, BucketFactory from apiver_deps import DownloadedFile from apiver_deps import DownloadVersion from apiver_deps import LargeFileUploadState from apiver_deps import MetadataDirectiveMode from apiver_deps import Part from apiver_deps import AbstractProgressListener from apiver_deps import StubAccountInfo, RawSimulator, BucketSimulator, FakeResponse, FileSimulator from apiver_deps import AbstractDownloader from apiver_deps import ParallelDownloader from apiver_deps import Range from apiver_deps import SimpleDownloader from apiver_deps import UploadSourceBytes from apiver_deps import hex_sha1_of_bytes, TempDir from apiver_deps import EncryptionAlgorithm, EncryptionSetting, EncryptionMode, EncryptionKey, SSE_NONE, SSE_B2_AES from apiver_deps import CopySource, UploadSourceLocalFile, WriteIntent from apiver_deps import BucketRetentionSetting, FileRetentionSetting, LegalHold, RetentionMode, RetentionPeriod, \ NO_RETENTION_FILE_SETTING from apiver_deps import ReplicationConfiguration, ReplicationRule pytestmark = [pytest.mark.apiver(from_ver=1)] SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_key', key_id='some-id'), ) SSE_C_AES_NO_SECRET = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=None, key_id='some-id'), ) SSE_C_AES_2 = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_other_key', key_id='some-id-2'), ) SSE_C_AES_2_NO_SECRET = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=None, key_id='some-id-2'), ) SSE_C_AES_FROM_SERVER = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(key_id=None, secret=None), ) REPLICATION = ReplicationConfiguration( rules=[ ReplicationRule( destination_bucket_id='c5f35d53a90a7ea284fb0719', name='replication-us-west', ), ReplicationRule( destination_bucket_id='55f34d53a96a7ea284fb0719', name='replication-us-west-2', file_name_prefix='replica/', is_enabled=False, priority=255, ), ], source_key_id='10053d55ae26b790000000006', source_to_destination_key_mapping={ "10053d55ae26b790000000045": "10053d55ae26b790000000004", "10053d55ae26b790000000046": "10053d55ae26b790030000004", }, ) def write_file(path, data): with open(path, 'wb') as f: f.write(data) class StubProgressListener(AbstractProgressListener): """ Implementation of a progress listener that remembers what calls were made, and returns them as a short string to use in unit tests. For a total byte count of 100, and updates at 33 and 66, the returned string looks like: "100: 33 66" """ def __init__(self): self.total = None self.history = [] self.last_byte_count = 0 def get_history(self): return ' '.join(self.history) def set_total_bytes(self, total_byte_count): assert total_byte_count is not None assert self.total is None, 'set_total_bytes called twice' self.total = total_byte_count assert len(self.history) == 0, self.history self.history.append('%d:' % (total_byte_count,)) def bytes_completed(self, byte_count): self.last_byte_count = byte_count self.history.append(str(byte_count)) def is_valid(self, **kwargs): valid, _ = self.is_valid_reason(**kwargs) return valid def is_valid_reason( self, check_closed=True, check_progress=True, check_monotonic_progress=False ): progress_end = -1 if self.history[progress_end] == 'closed': progress_end = -2 # self.total != self.last_byte_count may be a consequence of non-monotonic # progress, so we want to check this first if check_monotonic_progress: prev = 0 for val in map(int, self.history[1:progress_end]): if val < prev: return False, 'non-monotonic progress' prev = val if self.total != self.last_byte_count: return False, 'total different than last_byte_count' if check_closed and self.history[-1] != 'closed': return False, 'no "closed" at the end of history' if check_progress and len(self.history[1:progress_end]) < 2: return False, 'progress in history has less than 2 entries' return True, '' def close(self): self.history.append('closed') class CanRetry(B2Error): """ An exception that can be retryable, or not. """ def __init__(self, can_retry): super(CanRetry, self).__init__(None, None, None, None, None) self.can_retry = can_retry def should_retry_upload(self): return self.can_retry def bucket_ls(bucket, *args, show_versions=False, **kwargs): if apiver_deps.V <= 1: ls_all_versions_kwarg = {'show_versions': show_versions} else: ls_all_versions_kwarg = {'latest_only': not show_versions} return bucket.ls(*args, **ls_all_versions_kwarg, **kwargs) class TestCaseWithBucket(TestBase): RAW_SIMULATOR_CLASS = RawSimulator def get_api(self): return B2Api( self.account_info, api_config=B2HttpApiConfig(_raw_api_class=self.RAW_SIMULATOR_CLASS) ) def setUp(self): self.bucket_name = 'my-bucket' self.account_info = StubAccountInfo() self.api = self.get_api() self.simulator = self.api.session.raw_api (self.account_id, self.master_key) = self.simulator.create_account() self.api.authorize_account('production', self.account_id, self.master_key) self.api_url = self.account_info.get_api_url() self.account_auth_token = self.account_info.get_account_auth_token() self.bucket = self.api.create_bucket('my-bucket', 'allPublic') self.bucket_id = self.bucket.id_ def bucket_ls(self, *args, show_versions=False, **kwargs): return bucket_ls(self.bucket, *args, show_versions=show_versions, **kwargs) def assertBucketContents(self, expected, *args, **kwargs): """ *args and **kwargs are passed to self.bucket_ls() """ actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls(*args, **kwargs) ] self.assertEqual(expected, actual) def _make_data(self, approximate_length): """ Generate a sequence of bytes to use in testing an upload. Don't repeat a short pattern, so we're sure that the different parts of a large file are actually different. Returns bytes. """ fragments = [] so_far = 0 while so_far < approximate_length: fragment = ('%d:' % so_far).encode('utf-8') so_far += len(fragment) fragments.append(fragment) return b''.join(fragments) def _check_file_contents(self, file_name, expected_contents): contents = self._download_file(file_name) self.assertEqual(expected_contents, contents) def _download_file(self, file_name): with FileSimulator.dont_check_encryption(): if apiver_deps.V <= 1: download = DownloadDestBytes() self.bucket.download_file_by_name(file_name, download) return download.get_bytes_written() else: with io.BytesIO() as bytes_io: downloaded_file = self.bucket.download_file_by_name(file_name) downloaded_file.save(bytes_io) return bytes_io.getvalue() class TestReauthorization(TestCaseWithBucket): def testCreateBucket(self): class InvalidAuthTokenWrapper: def __init__(self, original_function): self.__original_function = original_function self.__name__ = original_function.__name__ self.__called = False def __call__(self, *args, **kwargs): if self.__called: return self.__original_function(*args, **kwargs) self.__called = True raise InvalidAuthToken('message', 401) self.simulator.create_bucket = InvalidAuthTokenWrapper(self.simulator.create_bucket) self.bucket = self.api.create_bucket('your-bucket', 'allPublic') class TestListParts(TestCaseWithBucket): @pytest.mark.apiver(to_ver=1) def testEmpty(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) self.assertEqual([], list(self.bucket.list_parts(file1.file_id, batch_size=1))) @pytest.mark.apiver(to_ver=1) def testThree(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) content = b'hello world' content_sha1 = hex_sha1_of_bytes(content) large_file_upload_state = mock.MagicMock() large_file_upload_state.has_error.return_value = False self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 1, large_file_upload_state ).result() self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 2, large_file_upload_state ).result() self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 3, large_file_upload_state ).result() expected_parts = [ Part('9999', 1, 11, content_sha1), Part('9999', 2, 11, content_sha1), Part('9999', 3, 11, content_sha1), ] self.assertEqual(expected_parts, list(self.bucket.list_parts(file1.file_id, batch_size=1))) class TestUploadPart(TestCaseWithBucket): @pytest.mark.apiver(to_ver=1) def test_error_in_state(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) content = b'hello world' file_progress_listener = mock.MagicMock() large_file_upload_state = LargeFileUploadState(file_progress_listener) large_file_upload_state.set_error('test error') try: self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 1, large_file_upload_state ).result() self.fail('should have thrown') except AlreadyFailed: pass class TestListUnfinished(TestCaseWithBucket): def test_empty(self): self.assertEqual([], list(self.bucket.list_unfinished_large_files())) @pytest.mark.apiver(to_ver=1) def test_one(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) self.assertEqual([file1], list(self.bucket.list_unfinished_large_files())) @pytest.mark.apiver(to_ver=1) def test_three(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) file2 = self.bucket.start_large_file('file2.txt', 'text/plain', {}) file3 = self.bucket.start_large_file('file3.txt', 'text/plain', {}) self.assertEqual( [file1, file2, file3], list(self.bucket.list_unfinished_large_files(batch_size=1)) ) @pytest.mark.apiver(to_ver=1) def test_prefix(self): self.bucket.start_large_file('fileA', 'text/plain', {}) file2 = self.bucket.start_large_file('fileAB', 'text/plain', {}) file3 = self.bucket.start_large_file('fileABC', 'text/plain', {}) self.assertEqual( [file2, file3], list(self.bucket.list_unfinished_large_files( batch_size=1, prefix='fileAB', ),), ) def _make_file(self, file_id, file_name): return self.bucket.start_large_file(file_name, 'text/plain', {}) class TestGetFileInfo(TestCaseWithBucket): def test_version_by_name(self): data = b'hello world' a_id = self.bucket.upload_bytes(data, 'a').id_ info = self.bucket.get_file_info_by_name('a') if apiver_deps.V <= 1: self.assertIsInstance(info, VFileVersionInfo) else: self.assertIsInstance(info, DownloadVersion) expected = (a_id, 'a', 11, 'b2/x-auto', 'none', NO_RETENTION_FILE_SETTING, LegalHold.UNSET) actual = ( info.id_, info.file_name, info.size, info.content_type, info.server_side_encryption.mode.value, info.file_retention, info.legal_hold, ) self.assertEqual(expected, actual) def test_version_by_name_file_lock(self): bucket = self.api.create_bucket( 'my-bucket-with-file-lock', 'allPublic', is_file_lock_enabled=True ) data = b'hello world' legal_hold = LegalHold.ON file_retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 100) bucket.upload_bytes(data, 'a', file_retention=file_retention, legal_hold=legal_hold) file_version = bucket.get_file_info_by_name('a') actual = (file_version.legal_hold, file_version.file_retention) self.assertEqual((legal_hold, file_retention), actual) low_perm_account_info = StubAccountInfo() low_perm_api = B2Api(low_perm_account_info) low_perm_api.session.raw_api = self.simulator low_perm_key = create_key( self.api, key_name='lowperm', capabilities=[ 'listKeys', 'listBuckets', 'listFiles', 'readFiles', ] ) low_perm_api.authorize_account('production', low_perm_key.id_, low_perm_key.application_key) low_perm_bucket = low_perm_api.get_bucket_by_name('my-bucket-with-file-lock') file_version = low_perm_bucket.get_file_info_by_name('a') actual = (file_version.legal_hold, file_version.file_retention) expected = (LegalHold.UNKNOWN, FileRetentionSetting(RetentionMode.UNKNOWN)) self.assertEqual(expected, actual) def test_version_by_id(self): data = b'hello world' b_id = self.bucket.upload_bytes(data, 'b').id_ info = self.bucket.get_file_info_by_id(b_id) self.assertIsInstance(info, VFileVersionInfo) expected = (b_id, 'b', 11, 'upload', 'b2/x-auto', 'none') actual = ( info.id_, info.file_name, info.size, info.action, info.content_type, info.server_side_encryption.mode.value ) self.assertEqual(expected, actual) class TestLs(TestCaseWithBucket): def test_empty(self): self.assertEqual([], list(self.bucket_ls('foo'))) def test_one_file_at_root(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '') def test_three_files_at_root(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb') self.bucket.upload_bytes(data, 'ccc') expected = [ ('a', 11, 'upload', None), ('bb', 11, 'upload', None), ('ccc', 11, 'upload', None) ] self.assertBucketContents(expected, '') def test_three_files_in_dir(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb/1') self.bucket.upload_bytes(data, 'bb/2/sub1') self.bucket.upload_bytes(data, 'bb/2/sub2') self.bucket.upload_bytes(data, 'bb/3') self.bucket.upload_bytes(data, 'ccc') expected = [ ('bb/1', 11, 'upload', None), ('bb/2/sub1', 11, 'upload', 'bb/2/'), ('bb/3', 11, 'upload', None) ] self.assertBucketContents(expected, 'bb', fetch_count=1) def test_three_files_multiple_versions(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb/1') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/3') self.bucket.upload_bytes(data, 'ccc') expected = [ ('9998', 'bb/1', 11, 'upload', None), ('9995', 'bb/2', 11, 'upload', None), ('9996', 'bb/2', 11, 'upload', None), ('9997', 'bb/2', 11, 'upload', None), ('9994', 'bb/3', 11, 'upload', None), ] actual = [ (info.id_, info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('bb', show_versions=True, fetch_count=1) ] self.assertEqual(expected, actual) @pytest.mark.apiver(to_ver=1) def test_started_large_file(self): self.bucket.start_large_file('hello.txt') expected = [('hello.txt', 0, 'start', None)] self.assertBucketContents(expected, '', show_versions=True) def test_hidden_file(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') self.bucket.hide_file('hello.txt') expected = [('hello.txt', 0, 'hide', None), ('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_delete_file_version(self): data = b'hello world' file_id = self.bucket.upload_bytes(data, 'hello.txt').id_ data = b'hello new world' self.bucket.upload_bytes(data, 'hello.txt') self.bucket.delete_file_version(file_id, 'hello.txt') expected = [('hello.txt', 15, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) class TestGetFreshState(TestCaseWithBucket): def test_ok(self): same_but_different = self.api.get_bucket_by_id(self.bucket.id_) same_but_different = same_but_different.get_fresh_state() assert isinstance(same_but_different, Bucket) assert id(same_but_different) != id(self.bucket) assert same_but_different.as_dict() == self.bucket.as_dict() same_but_different = same_but_different.update(bucket_info={'completely': 'new info'}) if apiver_deps.V <= 1: same_but_different = BucketFactory.from_api_bucket_dict(self.api, same_but_different) assert same_but_different.as_dict() != self.bucket.as_dict() refreshed_bucket = self.bucket.get_fresh_state() assert same_but_different.as_dict() == refreshed_bucket.as_dict() def test_fail(self): self.api.delete_bucket(self.bucket) with pytest.raises(BucketIdNotFound): self.bucket.get_fresh_state() class TestListVersions(TestCaseWithBucket): def test_single_version(self): data = b'hello world' a_id = self.bucket.upload_bytes(data, 'a').id_ b_id = self.bucket.upload_bytes(data, 'b').id_ c_id = self.bucket.upload_bytes(data, 'c').id_ expected = [(a_id, 'a', 11, 'upload')] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('a') ] self.assertEqual(expected, actual) expected = [(b_id, 'b', 11, 'upload')] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('b') ] self.assertEqual(expected, actual) expected = [(c_id, 'c', 11, 'upload')] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('c') ] self.assertEqual(expected, actual) def test_multiple_version(self): a_id1 = self.bucket.upload_bytes(b'first version', 'a').id_ a_id2 = self.bucket.upload_bytes(b'second version', 'a').id_ a_id3 = self.bucket.upload_bytes(b'last version', 'a').id_ expected = [ (a_id3, 'a', 12, 'upload'), (a_id2, 'a', 14, 'upload'), (a_id1, 'a', 13, 'upload') ] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('a') ] self.assertEqual(expected, actual) def test_ignores_subdirectory(self): data = b'hello world' file_id = self.bucket.upload_bytes(data, 'a/b').id_ self.bucket.upload_bytes(data, 'a/b/c') expected = [(file_id, 'a/b', 11, 'upload')] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('a/b') ] self.assertEqual(expected, actual) def test_all_versions_in_response(self): data = b'hello world' file_id = self.bucket.upload_bytes(data, 'a/b').id_ self.bucket.upload_bytes(data, 'a/b/c') expected = [(file_id, 'a/b', 11, 'upload')] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('a/b', fetch_count=1) ] self.assertEqual(expected, actual) def test_bad_fetch_count(self): try: # Convert to a list to cause the generator to execute. list(self.bucket.list_file_versions('a', fetch_count=0)) self.fail('should have raised ValueError') except ValueError as e: self.assertEqual('unsupported fetch_count value', str(e)) def test_encryption(self): data = b'hello world' a = self.bucket.upload_bytes(data, 'a') a_id = a.id_ self.assertEqual(a.server_side_encryption, SSE_NONE) b = self.bucket.upload_bytes(data, 'b', encryption=SSE_B2_AES) self.assertEqual(b.server_side_encryption, SSE_B2_AES) b_id = b.id_ # c_id = self.bucket.upload_bytes(data, 'c', encryption=SSE_NONE).id_ # TODO self.bucket.copy(a_id, 'd', destination_encryption=SSE_B2_AES) self.bucket.copy( b_id, 'e', destination_encryption=SSE_C_AES, file_info={}, content_type='text/plain' ) actual = [info.server_side_encryption for info in self.bucket.list_file_versions('a')][0] self.assertEqual(SSE_NONE, actual) # bucket default actual = self.bucket.get_file_info_by_name('a').server_side_encryption self.assertEqual(SSE_NONE, actual) # bucket default actual = [info.server_side_encryption for info in self.bucket.list_file_versions('b')][0] self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 actual = self.bucket.get_file_info_by_name('b').server_side_encryption self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 # actual = [info.server_side_encryption for info in self.bucket.list_file_versions('c')][0] # self.assertEqual(SSE_NONE, actual) # explicitly requested none actual = [info.server_side_encryption for info in self.bucket.list_file_versions('d')][0] self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 actual = self.bucket.get_file_info_by_name('d').server_side_encryption self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 actual = [info.server_side_encryption for info in self.bucket.list_file_versions('e')][0] self.assertEqual(SSE_C_AES_NO_SECRET, actual) # explicitly requested sse-c actual = self.bucket.get_file_info_by_name('e').server_side_encryption self.assertEqual(SSE_C_AES_NO_SECRET, actual) # explicitly requested sse-c class TestCopyFile(TestCaseWithBucket): @classmethod def _copy_function(cls, bucket): if apiver_deps.V <= 1: return bucket.copy_file else: return bucket.copy @pytest.mark.apiver(from_ver=2) def test_copy_big(self): data = b'HelloWorld' * 100 for i in range(10): data += bytes(':#' + str(i) + '$' + 'abcdefghijklmnopqrstuvwx' * 4, 'ascii') file_info = self.bucket.upload_bytes(data, 'file1') self.bucket.copy( file_info.id_, 'file2', min_part_size=200, max_part_size=400, ) self._check_file_contents('file2', data) def test_copy_without_optional_params(self): file_id = self._make_file() if apiver_deps.V <= 1: f = self.bucket.copy_file(file_id, 'hello_new.txt') assert f['action'] == 'copy' else: f = self.bucket.copy(file_id, 'hello_new.txt') assert f.action == 'copy' expected = [('hello.txt', 11, 'upload', None), ('hello_new.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_range(self): file_id = self._make_file() #data = b'hello world' # 3456789 if apiver_deps.V <= 1: self.bucket.copy_file( file_id, 'hello_new.txt', bytes_range=(3, 9), ) # inclusive, confusingly else: self.bucket.copy(file_id, 'hello_new.txt', offset=3, length=7) self._check_file_contents('hello_new.txt', b'lo worl') expected = [('hello.txt', 11, 'upload', None), ('hello_new.txt', 7, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) @pytest.mark.apiver(to_ver=1) def test_copy_with_invalid_metadata(self): file_id = self._make_file() try: self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.COPY, content_type='application/octet-stream', ) self.fail('should have raised InvalidMetadataDirective') except InvalidMetadataDirective as e: self.assertEqual( 'content_type and file_info should be None when metadata_directive is COPY', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) @pytest.mark.apiver(to_ver=1) def test_copy_with_invalid_metadata_replace(self): file_id = self._make_file() try: self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.REPLACE, ) self.fail('should have raised InvalidMetadataDirective') except InvalidMetadataDirective as e: self.assertEqual( 'content_type cannot be None when metadata_directive is REPLACE', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) @pytest.mark.apiver(to_ver=1) def test_copy_with_replace_metadata(self): file_id = self._make_file() self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.REPLACE, content_type='text/plain', ) expected = [ ('hello.txt', 11, 'upload', 'b2/x-auto', None), ('hello_new.txt', 11, 'upload', 'text/plain', None), ] actual = [ (info.file_name, info.size, info.action, info.content_type, folder) for (info, folder) in self.bucket_ls(show_versions=True) ] self.assertEqual(expected, actual) def test_copy_with_unsatisfied_range(self): file_id = self._make_file() try: if apiver_deps.V <= 1: self.bucket.copy_file( file_id, 'hello_new.txt', bytes_range=(12, 15), ) else: self.bucket.copy( file_id, 'hello_new.txt', offset=12, length=3, ) self.fail('should have raised UnsatisfiableRange') except UnsatisfiableRange as e: self.assertEqual( 'The range in the request is outside the size of the file', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_different_bucket(self): source_bucket = self.api.create_bucket('source-bucket', 'allPublic') file_id = self._make_file(source_bucket) self._copy_function(self.bucket)(file_id, 'hello_new.txt') def ls(bucket): return [ (info.file_name, info.size, info.action, folder) for (info, folder) in bucket_ls(bucket, show_versions=True) ] expected = [('hello.txt', 11, 'upload', None)] self.assertEqual(expected, ls(source_bucket)) expected = [('hello_new.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_retention(self): for data in [self._make_data(self.simulator.MIN_PART_SIZE * 3), b'hello']: for length in [None, len(data)]: with self.subTest(real_length=len(data), length=length): file_id = self.bucket.upload_bytes(data, 'original_file').id_ resulting_file_version = self.bucket.copy( file_id, 'copied_file', file_retention=FileRetentionSetting(RetentionMode.COMPLIANCE, 100), legal_hold=LegalHold.ON, max_part_size=400, ) self.assertEqual( FileRetentionSetting(RetentionMode.COMPLIANCE, 100), resulting_file_version.file_retention ) self.assertEqual(LegalHold.ON, resulting_file_version.legal_hold) def test_copy_encryption(self): data = b'hello_world' a = self.bucket.upload_bytes(data, 'a') a_id = a.id_ self.assertEqual(a.server_side_encryption, SSE_NONE) b = self.bucket.upload_bytes(data, 'b', encryption=SSE_B2_AES) self.assertEqual(b.server_side_encryption, SSE_B2_AES) b_id = b.id_ c = self.bucket.upload_bytes(data, 'c', encryption=SSE_C_AES) self.assertEqual(c.server_side_encryption, SSE_C_AES_NO_SECRET) c_id = c.id_ for length in [None, len(data)]: for kwargs, expected_encryption in [ (dict(file_id=a_id, destination_encryption=SSE_B2_AES), SSE_B2_AES), ( dict( file_id=a_id, destination_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain' ), SSE_C_AES_NO_SECRET ), ( dict( file_id=a_id, destination_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain' ), SSE_C_AES_NO_SECRET ), (dict(file_id=b_id), SSE_NONE), (dict(file_id=b_id, source_encryption=SSE_B2_AES), SSE_NONE), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_B2_AES ), SSE_B2_AES ), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain' ), SSE_C_AES_NO_SECRET ), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain' ), SSE_C_AES_NO_SECRET ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain' ), SSE_NONE ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain' ), SSE_NONE ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES ), SSE_C_AES_NO_SECRET ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_B2_AES, source_file_info={'old': 'value'}, source_content_type='text/plain' ), SSE_B2_AES ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_B2_AES, file_info={'new': 'value'}, content_type='text/plain' ), SSE_B2_AES ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES_2, source_file_info={'old': 'value'}, source_content_type='text/plain' ), SSE_C_AES_2_NO_SECRET ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES_2, file_info={'new': 'value'}, content_type='text/plain' ), SSE_C_AES_2_NO_SECRET ), ]: with self.subTest(kwargs=kwargs, length=length, data=data): file_info = self.bucket.copy(**kwargs, new_file_name='new_file', length=length) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(file_info.server_side_encryption, expected_encryption) def _make_file(self, bucket=None): data = b'hello world' actual_bucket = bucket or self.bucket return actual_bucket.upload_bytes(data, 'hello.txt').id_ class TestUpdate(TestCaseWithBucket): def test_update(self): result = self.bucket.update( bucket_type='allPrivate', bucket_info={'info': 'o'}, cors_rules={'andrea': 'corr'}, lifecycle_rules=[{ 'life': 'is life' }], default_server_side_encryption=SSE_B2_AES, default_retention=BucketRetentionSetting( RetentionMode.COMPLIANCE, RetentionPeriod(years=7) ), replication=REPLICATION, ) if apiver_deps.V <= 1: self.maxDiff = None with suppress(KeyError): del result['replicationConfiguration'] self.assertEqual( { 'accountId': 'account-0', 'bucketId': 'bucket_0', 'bucketInfo': { 'info': 'o' }, 'bucketName': 'my-bucket', 'bucketType': 'allPrivate', 'corsRules': { 'andrea': 'corr' }, 'defaultServerSideEncryption': { 'isClientAuthorizedToRead': True, 'value': { 'algorithm': 'AES256', 'mode': 'SSE-B2' } }, 'fileLockConfiguration': { 'isClientAuthorizedToRead': True, 'value': { 'defaultRetention': { 'mode': 'compliance', 'period': { 'unit': 'years', 'duration': 7 } }, 'isFileLockEnabled': None } }, 'lifecycleRules': [{ 'life': 'is life' }], 'options': set(), 'revision': 2 }, result ) else: self.assertIsInstance(result, Bucket) assertions_mapping = { # yapf: disable 'id_': self.bucket.id_, 'name': self.bucket.name, 'type_': 'allPrivate', 'bucket_info': {'info': 'o'}, 'cors_rules': {'andrea': 'corr'}, 'lifecycle_rules': [{'life': 'is life'}], 'options_set': set(), 'default_server_side_encryption': SSE_B2_AES, 'default_retention': BucketRetentionSetting(RetentionMode.COMPLIANCE, RetentionPeriod(years=7)), 'replication': REPLICATION, } for attr_name, attr_value in assertions_mapping.items(): self.maxDiff = None print('---', attr_name, '---') print(attr_value) print('?=?') print(getattr(result, attr_name)) self.assertEqual(attr_value, getattr(result, attr_name), attr_name) @pytest.mark.apiver(from_ver=2) def test_empty_replication(self): self.bucket.update( replication=ReplicationConfiguration( rules=[], source_to_destination_key_mapping={}, ), ) def test_update_if_revision_is(self): current_revision = self.bucket.revision self.bucket.update( lifecycle_rules=[{ 'life': 'is life' }], if_revision_is=current_revision, ) updated_bucket = self.api.get_bucket_by_name(self.bucket.name) self.assertEqual([{'life': 'is life'}], updated_bucket.lifecycle_rules) try: self.bucket.update( lifecycle_rules=[{ 'another': 'life' }], if_revision_is=current_revision, # this is now the old revision ) except Exception: pass not_updated_bucket = self.api.get_bucket_by_name(self.bucket.name) self.assertEqual([{'life': 'is life'}], not_updated_bucket.lifecycle_rules) class TestUpload(TestCaseWithBucket): def test_upload_bytes(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1') self.assertTrue(isinstance(file_info, VFileVersionInfo)) self._check_file_contents('file1', data) self.assertEqual(file_info.server_side_encryption, SSE_NONE) def test_upload_bytes_file_retention(self): data = b'hello world' retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 150) file_info = self.bucket.upload_bytes( data, 'file1', file_retention=retention, legal_hold=LegalHold.ON ) self._check_file_contents('file1', data) self.assertEqual(retention, file_info.file_retention) self.assertEqual(LegalHold.ON, file_info.legal_hold) def test_upload_bytes_sse_b2(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1', encryption=SSE_B2_AES) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(file_info.server_side_encryption, SSE_B2_AES) def test_upload_bytes_sse_c(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1', encryption=SSE_C_AES) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(SSE_C_AES_NO_SECRET, file_info.server_side_encryption) def test_upload_local_file_sse_b2(self): with TempDir() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1', encryption=SSE_B2_AES) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(file_info.server_side_encryption, SSE_B2_AES) self._check_file_contents('file1', data) def test_upload_local_file_sse_c(self): with TempDir() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1', encryption=SSE_C_AES) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(SSE_C_AES_NO_SECRET, file_info.server_side_encryption) self._check_file_contents('file1', data) def test_upload_local_file_retention(self): with TempDir() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 150) file_info = self.bucket.upload_local_file( path, 'file1', encryption=SSE_C_AES, file_retention=retention, legal_hold=LegalHold.ON ) self._check_file_contents('file1', data) self.assertEqual(retention, file_info.file_retention) self.assertEqual(LegalHold.ON, file_info.legal_hold) def test_upload_bytes_progress(self): data = b'hello world' progress_listener = StubProgressListener() self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertTrue(progress_listener.is_valid()) def test_upload_local_file(self): with TempDir() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1') self._check_file_contents('file1', data) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(file_info.server_side_encryption, SSE_NONE) print(file_info.as_dict()) self.assertEqual(file_info.as_dict()['serverSideEncryption'], {'mode': 'none'}) @pytest.mark.skipif(platform.system() == 'Windows', reason='no os.mkfifo() on Windows') def test_upload_fifo(self): with TempDir() as d: path = os.path.join(d, 'file1') os.mkfifo(path) with self.assertRaises(InvalidUploadSource): self.bucket.upload_local_file(path, 'file1') @pytest.mark.skipif(platform.system() == 'Windows', reason='no os.symlink() on Windows') def test_upload_dead_symlink(self): with TempDir() as d: path = os.path.join(d, 'file1') os.symlink('non-existing', path) with self.assertRaises(InvalidUploadSource): self.bucket.upload_local_file(path, 'file1') def test_upload_local_wrong_sha(self): with TempDir() as d: path = os.path.join(d, 'file123') data = b'hello world' write_file(path, data) with self.assertRaises(FileSha1Mismatch): self.bucket.upload_local_file( path, 'file123', sha1_sum='abcd' * 10, ) def test_upload_one_retryable_error(self): self.simulator.set_upload_errors([CanRetry(True)]) data = b'hello world' self.bucket.upload_bytes(data, 'file1') def test_upload_timeout(self): self.simulator.set_upload_errors([B2RequestTimeoutDuringUpload()]) data = b'hello world' self.bucket.upload_bytes(data, 'file1') def test_upload_file_one_fatal_error(self): self.simulator.set_upload_errors([CanRetry(False)]) data = b'hello world' with self.assertRaises(CanRetry): self.bucket.upload_bytes(data, 'file1') def test_upload_file_too_many_retryable_errors(self): self.simulator.set_upload_errors([CanRetry(True)] * 6) data = b'hello world' with self.assertRaises(MaxRetriesExceeded): self.bucket.upload_bytes(data, 'file1') def test_upload_large(self): data = self._make_data(self.simulator.MIN_PART_SIZE * 3) progress_listener = StubProgressListener() self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_local_large_file(self): with TempDir() as d: path = os.path.join(d, 'file1') data = self._make_data(self.simulator.MIN_PART_SIZE * 3) write_file(path, data) self.bucket.upload_local_file(path, 'file1') self._check_file_contents('file1', data) def test_upload_local_large_file_over_10k_parts(self): pytest.skip('this test is really slow and impedes development') # TODO: fix it with TempDir() as d: path = os.path.join(d, 'file1') data = self._make_data(self.simulator.MIN_PART_SIZE * 10001) # 2MB on the simulator write_file(path, data) self.bucket.upload_local_file(path, 'file1') self._check_file_contents('file1', data) def test_create_file_over_10k_parts(self): data = b'hello world' * 20000 f1_id = self.bucket.upload_bytes(data, 'f1').id_ with TempDir() as d: write_intents = [ WriteIntent( CopySource(f1_id, length=len(data), offset=0), destination_offset=0, ) ] * 10 created_file = self.bucket.create_file( write_intents, file_name='created_file', min_part_size=10, max_part_size=200, ) self.assertIsInstance(created_file, VFileVersionInfo) actual = ( created_file.id_, created_file.file_name, created_file.size, created_file.server_side_encryption, ) expected = ('9998', 'created_file', len(data), SSE_NONE) self.assertEqual(expected, actual) def test_upload_large_resume(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_no_parts(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertNotEqual(large_file_id, file_info.id_) # it's not a match if there are no parts self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_all_parts_there(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[:part_size]) self._upload_part(large_file_id, 2, data[part_size:2 * part_size]) self._upload_part(large_file_id, 3, data[2 * part_size:]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_part_does_not_match(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 3, data[:part_size]) # wrong part number for this data progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_wrong_part_size(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[:part_size + 1]) # one byte to much progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_file_info(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1', {'property': 'value1'}) self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes( data, 'file1', progress_listener=progress_listener, file_infos={'property': 'value1'} ) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_file_info_does_not_match(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1', {'property': 'value1'}) self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes( data, 'file1', progress_listener=progress_listener, file_infos={'property': 'value2'} ) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_file_with_restricted_api_key(self): self.simulator.key_id_to_key[self.account_id].name_prefix_or_none = 'path/to' part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes( data, 'path/to/file1', progress_listener=progress_listener ) self.assertEqual(len(data), file_info.size) self._check_file_contents('path/to/file1', data) self.assertTrue(progress_listener.is_valid()) def _start_large_file(self, file_name, file_info=None): if file_info is None: file_info = {} large_file_info = self.simulator.start_large_file( self.api_url, self.account_auth_token, self.bucket_id, file_name, None, file_info ) return large_file_info['fileId'] def _upload_part(self, large_file_id, part_number, part_data): part_stream = BytesIO(part_data) upload_info = self.simulator.get_upload_part_url( self.api_url, self.account_auth_token, large_file_id ) self.simulator.upload_part( upload_info['uploadUrl'], upload_info['authorizationToken'], part_number, len(part_data), hex_sha1_of_bytes(part_data), part_stream ) class TestConcatenate(TestCaseWithBucket): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.concatenate(sources, file_name=file_name, encryption=encryption) def test_create_remote(self): data = b'hello world' f1_id = self.bucket.upload_bytes(data, 'f1').id_ f2_id = self.bucket.upload_bytes(data, 'f1').id_ with TempDir() as d: path = os.path.join(d, 'file') write_file(path, data) created_file = self._create_remote( [ CopySource(f1_id, length=len(data), offset=0), UploadSourceLocalFile(path), CopySource(f2_id, length=len(data), offset=0), ], file_name='created_file' ) self.assertIsInstance(created_file, VFileVersionInfo) actual = ( created_file.id_, created_file.file_name, created_file.size, created_file.server_side_encryption ) expected = ('9997', 'created_file', 33, SSE_NONE) self.assertEqual(expected, actual) def test_create_remote_encryption(self): for data in [b'hello_world', self._make_data(self.simulator.MIN_PART_SIZE * 3)]: f1_id = self.bucket.upload_bytes(data, 'f1', encryption=SSE_C_AES).id_ f2_id = self.bucket.upload_bytes(data, 'f1', encryption=SSE_C_AES_2).id_ with TempDir() as d: path = os.path.join(d, 'file') write_file(path, data) created_file = self._create_remote( [ CopySource(f1_id, length=len(data), offset=0, encryption=SSE_C_AES), UploadSourceLocalFile(path), CopySource(f2_id, length=len(data), offset=0, encryption=SSE_C_AES_2), ], file_name='created_file_%s' % (len(data),), encryption=SSE_C_AES ) self.assertIsInstance(created_file, VFileVersionInfo) actual = ( created_file.id_, created_file.file_name, created_file.size, created_file.server_side_encryption ) expected = ( mock.ANY, 'created_file_%s' % (len(data),), mock.ANY, # FIXME: this should be equal to len(data) * 3, # but there is a problem in the simulator/test code somewhere SSE_C_AES_NO_SECRET ) self.assertEqual(expected, actual) class TestCreateFile(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.create_file( [wi for wi in WriteIntent.wrap_sources_iterator(sources)], file_name=file_name, encryption=encryption ) class TestConcatenateStream(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.concatenate_stream(sources, file_name=file_name, encryption=encryption) class TestCreateFileStream(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.create_file_stream( [wi for wi in WriteIntent.wrap_sources_iterator(sources)], file_name=file_name, encryption=encryption ) class DownloadTestsBase: DATA = NotImplemented def setUp(self): super(DownloadTestsBase, self).setUp() self.file_version = self.bucket.upload_bytes(self.DATA.encode(), 'file1') self.encrypted_file_version = self.bucket.upload_bytes( self.DATA.encode(), 'enc_file1', encryption=SSE_C_AES ) self.bytes_io = io.BytesIO() if apiver_deps.V <= 1: self.download_dest = DownloadDestBytes() else: self.download_dest = None self.progress_listener = StubProgressListener() def _verify(self, expected_result, check_progress_listener=True): self._assert_downloaded_data(expected_result) if check_progress_listener: valid, reason = self.progress_listener.is_valid_reason( check_closed=False, check_progress=False, check_monotonic_progress=True, ) assert valid, reason def _assert_downloaded_data(self, expected_result): if apiver_deps.V <= 1: assert self.download_dest.get_bytes_written() == expected_result.encode() else: assert self.bytes_io.getvalue() == expected_result.encode() def download_file_by_id(self, file_id, v1_download_dest=None, v2_file=None, **kwargs): if apiver_deps.V <= 1: self.bucket.download_file_by_id( file_id, v1_download_dest or self.download_dest, **kwargs ) else: self.bucket.download_file_by_id(file_id, **kwargs).save(v2_file or self.bytes_io) def download_file_by_name(self, file_name, download_dest=None, **kwargs): if apiver_deps.V <= 1: self.bucket.download_file_by_name( file_name, download_dest or self.download_dest, **kwargs ) else: self.bucket.download_file_by_name(file_name, **kwargs).save(self.bytes_io) class TestDownloadException(DownloadTestsBase, TestCaseWithBucket): DATA = 'some data' def test_download_file_by_name(self): if apiver_deps.V <= 1: exception_class = AssertionError else: exception_class = ValueError with mock.patch.object(self.bucket.api.services.download_manager, 'strategies', new=[]): with pytest.raises(exception_class) as exc_info: self.download_file_by_name(self.file_version.file_name) assert str(exc_info.value) == 'no strategy suitable for download was found!' class DownloadTests(DownloadTestsBase): DATA = 'abcdefghijklmnopqrs' def test_v2_return_types(self): download_kwargs = { 'range_': (7, 18), 'encryption': SSE_C_AES, 'progress_listener': self.progress_listener, } file_version = self.bucket.upload_bytes( self.DATA.encode(), 'enc_file2', encryption=SSE_C_AES ) other_properties = { 'download_version': DownloadVersion( api=self.api, id_=file_version.id_, file_name=file_version.file_name, size=len(self.DATA), content_type=file_version.content_type, content_sha1=file_version.content_sha1, file_info=file_version.file_info, upload_timestamp=file_version.upload_timestamp, server_side_encryption=file_version.server_side_encryption, range_=Range(7, 18), content_disposition=None, content_length=12, content_language=None, expires=None, cache_control=None, content_encoding=None, file_retention=file_version.file_retention, legal_hold=file_version.legal_hold, ), } ret = self.bucket.download_file_by_id(file_version.id_, **download_kwargs) assert isinstance(ret, DownloadedFile), type(ret) for attr_name, expected_value in {**download_kwargs, **other_properties}.items(): assert getattr(ret, attr_name) == expected_value, attr_name if apiver_deps.V >= 2: ret = self.bucket.download_file_by_name(file_version.file_name, **download_kwargs) assert isinstance(ret, DownloadedFile), type(ret) for attr_name, expected_value in {**download_kwargs, **other_properties}.items(): assert getattr(ret, attr_name) == expected_value, attr_name ret = file_version.download(**download_kwargs) assert isinstance(ret, DownloadedFile), type(ret) for attr_name, expected_value in {**download_kwargs, **other_properties}.items(): assert getattr(ret, attr_name) == expected_value, attr_name @pytest.mark.apiver(to_ver=1) def test_v1_return_types(self): expected = { 'contentLength': 19, 'contentSha1': '893e69ff0109f3459c4243013b3de8b12b41a30e', 'contentType': 'b2/x-auto', 'fileId': '9999', 'fileInfo': {}, 'fileName': 'file1' } ret = self.bucket.download_file_by_id(self.file_version.id_, self.download_dest) assert ret == expected ret = self.bucket.download_file_by_name(self.file_version.file_name, self.download_dest) assert ret == expected def test_download_file_version(self): self.file_version.download().save(self.bytes_io) assert self.bytes_io.getvalue() == self.DATA.encode() # self._verify performs different checks based on apiver, # but this is a new feature so it works the same on v2, v1 and v0 def test_download_by_id_no_progress(self): self.download_file_by_id(self.file_version.id_) self._verify(self.DATA, check_progress_listener=False) def test_download_by_name_no_progress(self): self.download_file_by_name('file1') self._verify(self.DATA, check_progress_listener=False) def test_download_by_name_progress(self): self.download_file_by_name('file1', progress_listener=self.progress_listener) self._verify(self.DATA) def test_download_by_id_progress(self): self.download_file_by_id(self.file_version.id_, progress_listener=self.progress_listener) self._verify(self.DATA) def test_download_by_id_progress_partial(self): self.download_file_by_id( self.file_version.id_, progress_listener=self.progress_listener, range_=(3, 9) ) self._verify('defghij') def test_download_by_id_progress_exact_range(self): self.download_file_by_id( self.file_version.id_, progress_listener=self.progress_listener, range_=(0, 18) ) self._verify(self.DATA) def test_download_by_id_progress_range_one_off(self): with self.assertRaises( InvalidRange, msg='A range of 0-19 was requested (size of 20), but cloud could only serve 19 of that', ): self.download_file_by_id( self.file_version.id_, progress_listener=self.progress_listener, range_=(0, 19), ) @pytest.mark.apiver(to_ver=1) def test_download_by_id_progress_partial_inplace_overwrite_v1(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # ||||||| # vvvvvvv # # 123defghij1234567890 with TempDir() as d: path = os.path.join(d, 'file2') download_dest = PreSeekedDownloadDest(seek_target=3, local_file_path=path) data = b'12345678901234567890' write_file(path, data) self.download_file_by_id( self.file_version.id_, download_dest, progress_listener=self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'123defghij1234567890') @pytest.mark.apiver(from_ver=2) def test_download_by_id_progress_partial_inplace_overwrite_v2(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # ||||||| # vvvvvvv # # 123defghij1234567890 with TempDir() as d: path = os.path.join(d, 'file2') data = b'12345678901234567890' write_file(path, data) with io.open(path, 'rb+') as file: file.seek(3) self.download_file_by_id( self.file_version.id_, v2_file=file, progress_listener=self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'123defghij1234567890') @pytest.mark.apiver(from_ver=2) def test_download_update_mtime_v2(self): with TempDir() as d: file_version = self.bucket.upload_bytes( self.DATA.encode(), 'file1', file_infos={'src_last_modified_millis': '1000'} ) path = os.path.join(d, 'file2') self.bucket.download_file_by_id(file_version.id_).save_to(path) assert pytest.approx(1, rel=0.001) == os.path.getmtime(path) @pytest.mark.apiver(to_ver=1) def test_download_by_id_progress_partial_shifted_overwrite_v1(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # ||||||| # vvvvvvv # # 1234567defghij567890 with TempDir() as d: path = os.path.join(d, 'file2') download_dest = PreSeekedDownloadDest(seek_target=7, local_file_path=path) data = b'12345678901234567890' write_file(path, data) self.download_file_by_id( self.file_version.id_, download_dest, progress_listener=self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'1234567defghij567890') @pytest.mark.apiver(from_ver=2) def test_download_by_id_progress_partial_shifted_overwrite_v2(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # ||||||| # vvvvvvv # # 1234567defghij567890 with TempDir() as d: path = os.path.join(d, 'file2') data = b'12345678901234567890' write_file(path, data) with io.open(path, 'rb+') as file: file.seek(7) self.download_file_by_id( self.file_version.id_, v2_file=file, progress_listener=self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'1234567defghij567890') def test_download_by_id_no_progress_encryption(self): self.download_file_by_id(self.encrypted_file_version.id_, encryption=SSE_C_AES) self._verify(self.DATA, check_progress_listener=False) def test_download_by_id_no_progress_wrong_encryption(self): with self.assertRaises(SSECKeyError): self.download_file_by_id(self.encrypted_file_version.id_, encryption=SSE_C_AES_2) def _check_local_file_contents(self, path, expected_contents): with open(path, 'rb') as f: contents = f.read() self.assertEqual(contents, expected_contents) # download empty file class EmptyFileDownloadScenarioMixin: """ use with DownloadTests, but not for TestDownloadParallel as it does not like empty files """ def test_download_by_name_empty_file(self): self.file_version = self.bucket.upload_bytes(b'', 'empty') self.download_file_by_name('empty', progress_listener=self.progress_listener) self._verify('') class UnverifiedChecksumDownloadScenarioMixin: """ use with DownloadTests """ def test_download_by_name_unverified_checksum(self): with TempDir() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1') simulated_file = list(self.simulator.bucket_name_to_bucket.values() )[0].file_id_to_file[file_info.id_] simulated_file.content_sha1 = 'unverified:' + simulated_file.content_sha1 #, sha1_sum='unverified:2aae6c35c94fcfb415dbe95f408b9ce91ee846ed') self.download_file_by_name('file1', progress_listener=self.progress_listener) self._verify('hello world') # actual tests # test choosing strategy @pytest.mark.apiver(from_ver=2) class TestChooseStrategy(TestCaseWithBucket): def test_choose_strategy(self): file_version = self.bucket.upload_bytes(b'hello world' * 8, 'file1') download_manager = self.bucket.api.services.download_manager parallel_downloader = ParallelDownloader( force_chunk_size=1, max_streams=32, min_part_size=16, thread_pool=download_manager._thread_pool, ) simple_downloader = download_manager.strategies[1] download_manager.strategies = [ parallel_downloader, simple_downloader, ] with io.BytesIO() as bytes_io: downloaded_file = self.bucket.download_file_by_id(file_version.id_) downloaded_file.save(bytes_io, allow_seeking=True) assert downloaded_file.download_strategy == parallel_downloader downloaded_file = self.bucket.download_file_by_id(file_version.id_) downloaded_file.save(bytes_io, allow_seeking=False) assert downloaded_file.download_strategy == simple_downloader downloaded_file = self.bucket.download_file_by_name(file_version.file_name) downloaded_file.save(bytes_io, allow_seeking=True) assert downloaded_file.download_strategy == parallel_downloader downloaded_file = self.bucket.download_file_by_name(file_version.file_name) downloaded_file.save(bytes_io, allow_seeking=False) assert downloaded_file.download_strategy == simple_downloader # Default tests class TestDownloadDefault(DownloadTests, EmptyFileDownloadScenarioMixin, TestCaseWithBucket): pass class TestDownloadSimple( DownloadTests, UnverifiedChecksumDownloadScenarioMixin, EmptyFileDownloadScenarioMixin, TestCaseWithBucket, ): def setUp(self): super(TestDownloadSimple, self).setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [SimpleDownloader(force_chunk_size=20)] class TestDownloadParallel( DownloadTests, UnverifiedChecksumDownloadScenarioMixin, TestCaseWithBucket, ): def setUp(self): super(TestDownloadParallel, self).setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [ ParallelDownloader( force_chunk_size=2, max_streams=999, min_part_size=2, ), ] class TestDownloadParallelALotOfStreams(DownloadTestsBase, TestCaseWithBucket): DATA = ''.join(['01234567890abcdef'] * 32) def setUp(self): super(TestDownloadParallelALotOfStreams, self).setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [ # this should produce 32 streams with 16 single byte writes # so we increase probability of non-sequential writes as much as possible # with great help from random sleeps in FakeResponse ParallelDownloader( force_chunk_size=1, max_streams=32, min_part_size=16, thread_pool=download_manager._thread_pool, ), ] def test_download_by_id_progress_monotonic(self): self.download_file_by_id(self.file_version.id_, progress_listener=self.progress_listener) self._verify(self.DATA) # Truncated downloads class TruncatedFakeResponse(FakeResponse): """ A special FakeResponse class which returns only the first 4 bytes of data. Use it to test followup retries for truncated download issues. """ def __init__(self, *args, **kwargs): super(TruncatedFakeResponse, self).__init__(*args, **kwargs) self.data_bytes = self.data_bytes[:4] class TruncatedDownloadBucketSimulator(BucketSimulator): RESPONSE_CLASS = TruncatedFakeResponse class TruncatedDownloadRawSimulator(RawSimulator): BUCKET_SIMULATOR_CLASS = TruncatedDownloadBucketSimulator class TestCaseWithTruncatedDownloadBucket(TestCaseWithBucket): RAW_SIMULATOR_CLASS = TruncatedDownloadRawSimulator ####### actual tests of truncated downloads class TestTruncatedDownloadSimple(DownloadTests, TestCaseWithTruncatedDownloadBucket): def setUp(self): super(TestTruncatedDownloadSimple, self).setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [SimpleDownloader(force_chunk_size=20)] class TestTruncatedDownloadParallel(DownloadTests, TestCaseWithTruncatedDownloadBucket): def setUp(self): super(TestTruncatedDownloadParallel, self).setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [ ParallelDownloader( force_chunk_size=3, max_streams=2, min_part_size=2, ) ] class DummyDownloader(AbstractDownloader): def download(self, *args, **kwargs): pass @pytest.mark.parametrize( 'min_chunk_size,max_chunk_size,content_length,align_factor,expected_chunk_size', [ (10, 100, 1000 * 9, 8, 8), # min_chunk_size aligned (10, 100, 1000 * 17, 8, 16), # content_length // 1000 aligned (10, 100, 1000 * 108, 8, 96), # max_chunk_size // 1000 aligned (10, 100, 1000 * 9, 100, 100), # max_chunk_size/align_factor (10, 100, 1000 * 17, 100, 100), # max_chunk_size/align_factor (10, 100, 1000 * 108, 100, 100), # max_chunk_size/align_factor (10, 100, 1, 100, 100), # max_chunk_size/align_factor ], ) def test_downloader_get_chunk_size( min_chunk_size, max_chunk_size, content_length, align_factor, expected_chunk_size ): downloader = DummyDownloader( min_chunk_size=min_chunk_size, max_chunk_size=max_chunk_size, align_factor=align_factor, ) assert downloader._get_chunk_size(content_length) == expected_chunk_size @pytest.mark.apiver(from_ver=2) class TestDownloadTuneWriteBuffer(DownloadTestsBase, TestCaseWithBucket): ALIGN_FACTOR = 123 DATA = 'abc' * 4096 def get_api(self): return B2Api( self.account_info, api_config=B2HttpApiConfig(_raw_api_class=self.RAW_SIMULATOR_CLASS), save_to_buffer_size=self.ALIGN_FACTOR, ) def test_get_chunk_size_alignment(self): download_manager = self.bucket.api.services.download_manager for downloader in download_manager.strategies: assert downloader._get_chunk_size(len(self.DATA)) % self.ALIGN_FACTOR == 0 def test_buffering_in_save_to(self): with TempDir() as d: path = os.path.join(d, 'file2') with mock.patch('b2sdk.transfer.inbound.downloaded_file.open') as mock_open: mock_open.side_effect = open self.bucket.download_file_by_id(self.file_version.id_).save_to(path) mock_open.assert_called_once_with(path, mock.ANY, buffering=self.ALIGN_FACTOR) with open(path, 'r') as f: contents = f.read() assert contents == self.DATA def test_set_write_buffer_parallel_called_get_chunk_size(self): self._check_called_on_downloader( ParallelDownloader( force_chunk_size=len(self.DATA) // 3, max_streams=999, min_part_size=len(self.DATA) // 3, ) ) def test_set_write_buffer_simple_called_get_chunk_size(self): self._check_called_on_downloader(SimpleDownloader(force_chunk_size=len(self.DATA) // 3)) def _check_called_on_downloader(self, downloader): download_manager = self.bucket.api.services.download_manager download_manager.strategies = [downloader] orig_get_chunk_size = downloader._get_chunk_size with mock.patch.object(downloader, '_get_chunk_size') as mock_get_chunk_size: mock_get_chunk_size.side_effect = orig_get_chunk_size self.download_file_by_id(self.file_version.id_) self._verify(self.DATA, check_progress_listener=False) assert mock_get_chunk_size.called @pytest.mark.apiver(from_ver=2) class TestDownloadNoHashChecking(DownloadTestsBase, TestCaseWithBucket): DATA = 'abcdefghijklmnopqrs' def get_api(self): return B2Api( self.account_info, api_config=B2HttpApiConfig(_raw_api_class=self.RAW_SIMULATOR_CLASS), check_download_hash=False, ) def test_download_by_id_no_hash_checking(self): downloaded_file = self.bucket.download_file_by_id(self.file_version.id_) orig_validate_download = downloaded_file._validate_download with mock.patch.object(downloaded_file, '_validate_download') as mocked_validate_download: mocked_validate_download.side_effect = orig_validate_download downloaded_file.save(self.bytes_io) self._verify(self.DATA, check_progress_listener=False) mocked_validate_download.assert_called_once_with(mock.ANY, '') assert downloaded_file.download_version.content_sha1 != 'none' assert downloaded_file.download_version.content_sha1 != '' class DecodeTestsBase(object): def setUp(self): super(DecodeTestsBase, self).setUp() self.bucket.upload_bytes( 'Test File 1'.encode(), 'test.txt?foo=bar', file_infos={'custom_info': 'aaa?bbb'} ) self.bucket.upload_bytes( 'Test File 2'.encode(), 'test.txt%3Ffoo=bar', file_infos={'custom_info': 'aaa%3Fbbb'} ) self.bucket.upload_bytes('Test File 3'.encode(), 'test.txt%3Ffoo%3Dbar') self.bucket.upload_bytes('Test File 4'.encode(), 'test.txt%253Ffoo%253Dbar') self.bytes_io = io.BytesIO() if apiver_deps.V <= 1: self.download_dest = DownloadDestBytes() else: self.download_dest = None self.progress_listener = StubProgressListener() def _verify(self, expected_result, check_progress_listener=True): self._assert_downloaded_data(expected_result) if check_progress_listener: valid, reason = self.progress_listener.is_valid_reason( check_closed=False, check_progress=False, check_monotonic_progress=True, ) assert valid, reason def _assert_downloaded_data(self, expected_result): if apiver_deps.V <= 1: assert self.download_dest.get_bytes_written() == expected_result.encode() else: assert self.bytes_io.getvalue() == expected_result.encode() def download_file_by_name(self, file_name, download_dest=None, **kwargs): if apiver_deps.V <= 1: self.bucket.download_file_by_name( file_name, download_dest or self.download_dest, **kwargs ) else: self.bucket.download_file_by_name(file_name, **kwargs).save(self.bytes_io) class DecodeTests(DecodeTestsBase, TestCaseWithBucket): def test_file_content_1(self): self.download_file_by_name('test.txt?foo=bar', progress_listener=self.progress_listener) self._verify("Test File 1") def test_file_content_2(self): self.download_file_by_name('test.txt%3Ffoo=bar', progress_listener=self.progress_listener) self._verify("Test File 2") def test_file_content_3(self): self.download_file_by_name('test.txt%3Ffoo%3Dbar', progress_listener=self.progress_listener) self._verify("Test File 3") def test_file_content_4(self): self.download_file_by_name( 'test.txt%253Ffoo%253Dbar', progress_listener=self.progress_listener ) self._verify("Test File 4") def test_file_info_1(self): download_version = self.bucket.get_file_info_by_name('test.txt?foo=bar') assert download_version.file_name == 'test.txt?foo=bar' assert download_version.file_info['custom_info'] == 'aaa?bbb' def test_file_info_2(self): download_version = self.bucket.get_file_info_by_name('test.txt%3Ffoo=bar') assert download_version.file_name == 'test.txt%3Ffoo=bar' assert download_version.file_info['custom_info'] == 'aaa%3Fbbb' def test_file_info_3(self): download_version = self.bucket.get_file_info_by_name('test.txt%3Ffoo%3Dbar') assert download_version.file_name == 'test.txt%3Ffoo%3Dbar' def test_file_info_4(self): download_version = self.bucket.get_file_info_by_name('test.txt%253Ffoo%253Dbar') assert download_version.file_name == 'test.txt%253Ffoo%253Dbar' b2-sdk-python-1.17.3/test/unit/conftest.py000066400000000000000000000102441426424117700203520ustar00rootroot00000000000000###################################################################### # # File: test/unit/conftest.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os import sys from glob import glob from pathlib import Path import pytest pytest.register_assert_rewrite('test.unit') def get_api_versions(): return [ str(Path(p).parent.name) for p in sorted(glob(str(Path(__file__).parent / 'v*/apiver/'))) ] API_VERSIONS = get_api_versions() @pytest.hookimpl def pytest_addoption(parser): """Add an argument for running test for given apiver.""" parser.addoption( '--api', default=API_VERSIONS[-1], choices=API_VERSIONS, help='version of the API', ) @pytest.hookimpl def pytest_configure(config): """Add apiver test folder to the path and add "apiver" marker used by `pytest_runtest_setup`.""" sys.path.insert(0, str(Path(__file__).parent / config.getoption('--api') / 'apiver')) config.addinivalue_line( 'markers', 'apiver(*args, *, from_ver=0, to_ver=sys.maxsize): mark test to run only for specific apivers' ) @pytest.hookimpl def pytest_report_header(config): """Print apiver in the header.""" return 'b2sdk apiver: %s' % config.getoption('--api') @pytest.hookimpl(tryfirst=True) def pytest_ignore_collect(path, config): """Ignore all tests from subfolders for different apiver.""" path = str(path) ver = config.getoption('--api') other_versions = [v for v in API_VERSIONS if v != ver] for other_version in other_versions: if other_version + os.sep in path: return True return False def pytest_runtest_setup(item): """ Skip tests based on "apiver" marker. .. code-block:: python @pytest.mark.apiver(1) def test_only_for_v1(self): ... @pytest.mark.apiver(1, 3) def test_only_for_v1_and_v3(self): ... @pytest.mark.apiver(from_ver=2) def test_for_greater_or_equal_v2(self): ... @pytest.mark.apiver(to_ver=2) def test_for_less_or_equal_v2(self): ... @pytest.mark.apiver(from_ver=2, to_ver=4) def test_for_versions_from_v2_to_v4(self): ... Both `from_ver` and `to_ver` are inclusive. Providing test parameters based on apiver is also possible: .. code-block:: python @pytest.mark.parametrize( 'exc_class,exc_msg', [ pytest.param(InvalidArgument, None, marks=pytest.mark.apiver(from_ver=2)), pytest.param(re.error, "invalid something", marks=pytest.mark.apiver(to_ver=1)), ], ) def test_illegal_regex(self, exc_class, exc_msg): with pytest.raises(exc_class, match=exc_msg): error_raising_function() If a test is merked with "apiver" multiple times, it will be skipped if at least one of the apiver conditions cause it to be skipped. E.g. if a test module is marked with .. code-block:: python pytestmark = [pytest.mark.apiver(from_ver=1)] and a test function is marked with .. code-block:: python @pytest.mark.apiver(to_ver=1) def test_function(self): ... the test function will be run only for apiver=v1 """ for mark in item.iter_markers(name='apiver'): if mark.args and mark.kwargs: raise pytest.UsageError('apiver mark should not have both args and kwargs') int_ver = int(item.config.getoption('--api')[1:]) if mark.args: if int_ver not in mark.args: pytest.skip('test requires apiver to be one of: %s' % mark.args) elif mark.kwargs: from_ver = mark.kwargs.get('from_ver', 0) to_ver = mark.kwargs.get('to_ver', sys.maxsize) if not (from_ver <= int_ver <= to_ver): pytest.skip('test requires apiver to be in range: [%d, %d]' % (from_ver, to_ver)) @pytest.fixture(scope='session') def apiver(request): """Get apiver as a v-prefixed string, e.g. "v2".""" return request.config.getoption('--api') b2-sdk-python-1.17.3/test/unit/file_version/000077500000000000000000000000001426424117700206365ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/file_version/__init__.py000066400000000000000000000004551426424117700227530ustar00rootroot00000000000000###################################################################### # # File: test/unit/file_version/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/file_version/test_file_version.py000066400000000000000000000166421426424117700247440ustar00rootroot00000000000000###################################################################### # # File: test/unit/file_version/test_file_version.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest import apiver_deps from apiver_deps import B2Api from apiver_deps import B2HttpApiConfig from apiver_deps import DownloadVersion from apiver_deps import DummyCache from apiver_deps import EncryptionAlgorithm from apiver_deps import EncryptionKey from apiver_deps import EncryptionMode from apiver_deps import EncryptionSetting from apiver_deps import FileIdAndName from apiver_deps import FileRetentionSetting from apiver_deps import InMemoryAccountInfo from apiver_deps import LegalHold from apiver_deps import RawSimulator from apiver_deps import RetentionMode from apiver_deps_exception import FileNotPresent if apiver_deps.V <= 1: from apiver_deps import FileVersionInfo as VFileVersion else: from apiver_deps import FileVersion as VFileVersion class TestFileVersion: @pytest.fixture(autouse=True) def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = DummyCache() self.api = B2Api( self.account_info, self.cache, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator) ) self.raw_api = self.api.session.raw_api (self.application_key_id, self.master_key) = self.raw_api.create_account() self.api.authorize_account('production', self.application_key_id, self.master_key) self.bucket = self.api.create_bucket('testbucket', 'allPrivate', is_file_lock_enabled=True) self.file_version = self.bucket.upload_bytes(b'nothing', 'test_file') @pytest.mark.apiver(to_ver=1) def test_format_ls_entry(self): file_version_info = VFileVersion( 'a2', 'inner/a.txt', 200, 'text/plain', 'sha1', {}, 2000, 'upload' ) expected_entry = ( ' ' ' a2 upload 1970-01-01 ' '00:00:02 200 inner/a.txt' ) assert expected_entry == file_version_info.format_ls_entry() def test_get_fresh_state(self): self.api.update_file_legal_hold( self.file_version.id_, self.file_version.file_name, LegalHold.ON ) fetched_version = self.api.get_file_info(self.file_version.id_) if apiver_deps.V <= 1: fetched_version = self.api.file_version_factory.from_api_response(fetched_version) assert self.file_version.as_dict() != fetched_version.as_dict() refreshed_version = self.file_version.get_fresh_state() assert isinstance(refreshed_version, VFileVersion) assert refreshed_version.as_dict() == fetched_version.as_dict() def test_clone_file_version_and_download_version(self): encryption = EncryptionSetting( EncryptionMode.SSE_C, EncryptionAlgorithm.AES256, EncryptionKey(b'secret', None) ) initial_file_version = self.bucket.upload_bytes( b'nothing', 'test_file', content_type='video/mp4', file_infos={ 'file': 'info', 'b2-content-language': 'en_US', 'b2-content-disposition': 'attachment', 'b2-expires': '2100-01-01', 'b2-cache-control': 'unknown', 'b2-content-encoding': 'text', }, encryption=encryption, file_retention=FileRetentionSetting(RetentionMode.GOVERNANCE, 100), legal_hold=LegalHold.ON, ) assert initial_file_version._clone() == initial_file_version cloned = initial_file_version._clone(legal_hold=LegalHold.OFF) assert isinstance(cloned, VFileVersion) assert cloned.as_dict() == { **initial_file_version.as_dict(), 'legalHold': LegalHold.OFF.value } download_version = self.api.download_file_by_id( initial_file_version.id_, encryption=encryption ).download_version assert download_version._clone() == download_version cloned = download_version._clone(legal_hold=LegalHold.OFF) assert isinstance(cloned, DownloadVersion) assert cloned.as_dict() == {**download_version.as_dict(), 'legalHold': LegalHold.OFF.value} def test_update_legal_hold(self): new_file_version = self.file_version.update_legal_hold(LegalHold.ON) assert isinstance(new_file_version, VFileVersion) assert new_file_version.legal_hold == LegalHold.ON download_version = self.api.download_file_by_id(self.file_version.id_).download_version new_download_version = download_version.update_legal_hold(LegalHold.ON) assert isinstance(new_download_version, DownloadVersion) assert new_download_version.legal_hold == LegalHold.ON def test_update_retention(self): new_retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 100) new_file_version = self.file_version.update_retention(new_retention) assert isinstance(new_file_version, VFileVersion) assert new_file_version.file_retention == new_retention download_version = self.api.download_file_by_id(self.file_version.id_).download_version new_download_version = download_version.update_retention(new_retention) assert isinstance(new_download_version, DownloadVersion) assert new_download_version.file_retention == new_retention def test_delete_file_version(self): ret = self.file_version.delete() assert isinstance(ret, FileIdAndName) with pytest.raises(FileNotPresent): self.bucket.get_file_info_by_name(self.file_version.file_name) def test_delete_download_version(self): download_version = self.api.download_file_by_id(self.file_version.id_).download_version ret = download_version.delete() assert isinstance(ret, FileIdAndName) with pytest.raises(FileNotPresent): self.bucket.get_file_info_by_name(self.file_version.file_name) def test_file_version_upload_headers(self): file_version = self.file_version._clone( server_side_encryption=EncryptionSetting( EncryptionMode.SSE_C, EncryptionAlgorithm.AES256, EncryptionKey(None, None), ), ) assert file_version._get_upload_headers() == """ Authorization: auth_token_0 Content-Length: 7 X-Bz-File-Name: test_file Content-Type: b2/x-auto X-Bz-Content-Sha1: 0feca720e2c29dafb2c900713ba560e03b758711 X-Bz-Server-Side-Encryption-Customer-Algorithm: AES256 X-Bz-Server-Side-Encryption-Customer-Key: KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKio= X-Bz-Server-Side-Encryption-Customer-Key-Md5: SaaDheEjzuynJH8eW6AEpQ== X-Bz-File-Legal-Hold: off X-Bz-File-Retention-Mode: None X-Bz-File-Retention-Retain-Until-Timestamp: None """.strip().replace(': ', '').replace(' ', '').replace('\n', '').encode('utf8') assert not file_version.has_large_header file_version.file_info['dummy'] = 'a' * 2000 # make metadata > 2k bytes assert file_version.has_large_header # FileVersion.download tests are not here, because another test file already has all the facilities for such test # prepared b2-sdk-python-1.17.3/test/unit/fixtures/000077500000000000000000000000001426424117700200235ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/fixtures/__init__.py000066400000000000000000000006031426424117700221330ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from .b2http import * from .cache import * from .raw_api import * from .session import * b2-sdk-python-1.17.3/test/unit/fixtures/b2http.py000066400000000000000000000006751426424117700216100ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/b2http.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from apiver_deps import B2Http @pytest.fixture def fake_b2http(mocker): return mocker.MagicMock(name='FakeB2Http', spec=B2Http) b2-sdk-python-1.17.3/test/unit/fixtures/cache.py000066400000000000000000000007101426424117700214360ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/cache.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from apiver_deps import InMemoryCache @pytest.fixture def fake_cache(mocker): return mocker.MagicMock(name='FakeCache', spec=InMemoryCache) b2-sdk-python-1.17.3/test/unit/fixtures/folder.py000066400000000000000000000061461426424117700216570ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/folder.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from unittest import mock import apiver_deps import pytest from apiver_deps import B2Folder, LocalFolder, LocalPath from apiver_deps import DEFAULT_SCAN_MANAGER if apiver_deps.V <= 1: from apiver_deps import FileVersionInfo as VFileVersion else: from apiver_deps import FileVersion as VFileVersion class FakeB2Folder(B2Folder): def __init__(self, test_files): self.file_versions = [] for test_file in test_files: self.file_versions.extend(self._file_versions(*test_file)) super().__init__('test-bucket', 'folder', mock.MagicMock()) def get_file_versions(self): yield from iter(self.file_versions) def _file_versions(self, name, mod_times, size=10): """ Makes FileVersion objects. Positive modification times are uploads, and negative modification times are hides. It's a hack, but it works. """ if apiver_deps.V <= 1: mandatory_kwargs = {} else: mandatory_kwargs = { 'api': None, 'account_id': 'account-id', 'bucket_id': 'bucket-id', 'content_md5': 'content_md5', 'server_side_encryption': None, } return [ VFileVersion( id_='id_%s_%d' % (name[0], abs(mod_time)), file_name='folder/' + name, upload_timestamp=abs(mod_time), action='upload' if 0 < mod_time else 'hide', size=size, file_info={'in_b2': 'yes'}, content_type='text/plain', content_sha1='content_sha1', **mandatory_kwargs, ) for mod_time in mod_times ] # yapf disable class FakeLocalFolder(LocalFolder): def __init__(self, test_files): super().__init__('folder') self.local_paths = [self._local_path(*test_file) for test_file in test_files] def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): for single_path in self.local_paths: if single_path.relative_path.endswith('/'): if policies_manager.should_exclude_b2_directory(single_path.relative_path): continue else: if policies_manager.should_exclude_local_path(single_path): continue yield single_path def make_full_path(self, name): return '/dir/' + name def _local_path(self, name, mod_times, size=10): """ Makes a LocalPath object for a local file. """ return LocalPath(name, name, mod_times[0], size) @pytest.fixture(scope='session') def folder_factory(): def get_folder(f_type, *files): if f_type == 'b2': return FakeB2Folder(files) return FakeLocalFolder(files) return get_folder b2-sdk-python-1.17.3/test/unit/fixtures/raw_api.py000066400000000000000000000027011426424117700220170ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/raw_api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from copy import copy import pytest from apiver_deps import ALL_CAPABILITIES, B2RawHTTPApi @pytest.fixture def fake_b2_raw_api_responses(): return { 'authorize_account': { 'absoluteMinimumPartSize': 5000000, 'accountId': '6012deadbeef', 'allowed': { 'bucketId': None, 'bucketName': None, 'capabilities': copy(ALL_CAPABILITIES), 'namePrefix': None, }, 'apiUrl': 'https://api000.backblazeb2.xyz:8180', 'authorizationToken': '4_1111111111111111111111111_11111111_111111_1111_1111111111111_1111_11111111=', 'downloadUrl': 'https://f000.backblazeb2.xyz:8180', 'recommendedPartSize': 100000000, 's3ApiUrl': 'https://s3.us-west-000.backblazeb2.xyz:8180', } } # yapf: disable @pytest.fixture def fake_b2_raw_api(mocker, fake_b2http, fake_b2_raw_api_responses): raw_api = mocker.MagicMock(name='FakeB2RawHTTPApi', spec=B2RawHTTPApi) raw_api.b2_http = fake_b2http raw_api.authorize_account.return_value = fake_b2_raw_api_responses['authorize_account'] return raw_api b2-sdk-python-1.17.3/test/unit/fixtures/session.py000066400000000000000000000015471426424117700220670ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/session.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from apiver_deps import B2Session @pytest.fixture def b2_session(fake_account_info, fake_cache, fake_b2_raw_api): session = B2Session(account_info=fake_account_info, cache=fake_cache) session.raw_api = fake_b2_raw_api return session @pytest.fixture def fake_b2_session(mocker, fake_account_info, fake_cache, fake_b2_raw_api): b2_session = mocker.MagicMock(name='FakeB2Session', spec=B2Session) b2_session.account_info = fake_account_info b2_session.cache = fake_cache b2_session.raw_api = fake_b2_raw_api return b2_session b2-sdk-python-1.17.3/test/unit/internal/000077500000000000000000000000001426424117700177665ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/internal/__init__.py000066400000000000000000000004511426424117700220770ustar00rootroot00000000000000###################################################################### # # File: test/unit/internal/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/internal/test_base.py000066400000000000000000000021741426424117700223150ustar00rootroot00000000000000###################################################################### # # File: test/unit/internal/test_base.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from contextlib import contextmanager import re import unittest class TestBase(unittest.TestCase): @contextmanager def assertRaises(self, exc, msg=None): try: yield except exc as e: if msg is not None: if msg != str(e): assert False, "expected message '%s', but got '%s'" % (msg, str(e)) else: assert False, 'should have thrown %s' % (exc,) @contextmanager def assertRaisesRegexp(self, expected_exception, expected_regexp): try: yield except expected_exception as e: if not re.search(expected_regexp, str(e)): assert False, "expected message '%s', but got '%s'" % (expected_regexp, str(e)) else: assert False, 'should have thrown %s' % (expected_exception,) b2-sdk-python-1.17.3/test/unit/internal/test_emerge_planner.py000066400000000000000000000555551426424117700244010ustar00rootroot00000000000000###################################################################### # # File: test/unit/internal/test_emerge_planner.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.transfer.emerge.planner.planner import ( EmergePlanner, GIGABYTE, MEGABYTE, ) from b2sdk.transfer.emerge.planner.part_definition import ( CopyEmergePartDefinition, UploadEmergePartDefinition, UploadSubpartsEmergePartDefinition, ) from b2sdk.transfer.emerge.planner.upload_subpart import ( LocalSourceUploadSubpart, RemoteSourceUploadSubpart, ) from b2sdk.transfer.emerge.write_intent import WriteIntent from b2sdk.transfer.outbound.copy_source import CopySource as OrigCopySource from b2sdk.transfer.outbound.upload_source import UploadSourceStream from .test_base import TestBase class UploadSource(UploadSourceStream): def __init__(self, length): super(UploadSource, self).__init__(lambda: None, length) class CopySource(OrigCopySource): def __init__(self, length): super(CopySource, self).__init__(id(self), length=length) def part(source_or_def_list, *offset_len): """ Helper for building emerge parts from outbound sources defs. Makes planner tests easier to read. Possible "def" structures: ``outbound_source`` - will build correct emerge part using type introspection with ``0`` for ``offset`` and ``outbound_source.get_content_length()`` for ``length`` ``outbound_source, offset, length`` - same as above, but given ``offset`` and ``length`` ``[outbound_source_def,...]``` - will build emerge part using ``UploadSubpartsEmergePartDefinition`` and type introspection for upload subparts; ``outbound_source_def`` is similar to above, either outbound source or tuple with source, offset, length Please notice that ``part([copy_source])`` is a single "small copy" - first download then upload, and ``part(copy_source) is "regular copy" - ``length`` is not verified here """ if isinstance(source_or_def_list, list): assert not offset_len subparts = [] for sub_part_meta in source_or_def_list: if isinstance(sub_part_meta, tuple): source, offset, length = sub_part_meta else: source = sub_part_meta offset = 0 length = source.get_content_length() if isinstance(source, UploadSource): subparts.append(LocalSourceUploadSubpart(source, offset, length)) else: subparts.append(RemoteSourceUploadSubpart(source, offset, length)) return UploadSubpartsEmergePartDefinition(subparts) else: source = source_or_def_list if offset_len: offset, length = offset_len else: offset = 0 length = source.get_content_length() if isinstance(source, UploadSource): return UploadEmergePartDefinition(source, offset, length) else: return CopyEmergePartDefinition(source, offset, length) class TestEmergePlanner(TestBase): def setUp(self): # we want to hardcode here current production settings and change them # whenever these are changed on production, and relate all test lengths # to those sizes - if test assumes something about those values # (like `min_part_size` > 2 * MEGABYTE), then one should add assertion # at the beginning of test function body self.recommended_size = 100 * MEGABYTE self.min_size = 5 * MEGABYTE self.max_size = 5 * GIGABYTE self.planner = self._get_emerge_planner() def _get_emerge_planner(self): return EmergePlanner( min_part_size=self.min_size, recommended_upload_part_size=self.recommended_size, max_part_size=self.max_size, ) # yapf: disable def test_part_sizes(self): self.assertGreater(self.min_size, 0) self.assertGreaterEqual(self.recommended_size, self.min_size) self.assertGreaterEqual(self.max_size, self.recommended_size) def test_simple_concatenate(self): sources = [ CopySource(self.recommended_size), UploadSource(self.recommended_size), CopySource(self.recommended_size), UploadSource(self.recommended_size), ] self.verify_emerge_plan_for_write_intents( WriteIntent.wrap_sources_iterator(sources), [part(source) for source in sources], ) def test_single_part_upload(self): source = UploadSource(self.recommended_size) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], [part(source)], ) def test_single_part_copy(self): source = CopySource(self.max_size) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], [part(source)], ) def test_single_multipart_upload(self): self.assertGreater(self.recommended_size, 2 * self.min_size) remainder = 2 * self.min_size source = UploadSource(self.recommended_size * 5 + remainder) expected_part_sizes = [self.recommended_size] * 5 + [remainder] self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], self.split_source_to_part_defs(source, expected_part_sizes), ) def test_recommended_part_size_decrease(self): source_upload1 = UploadSource(self.recommended_size * 10001) write_intents = [ WriteIntent(source_upload1), ] emerge_plan = self.planner.get_emerge_plan(write_intents) assert len(emerge_plan.emerge_parts) < 10000 def test_single_multipart_copy(self): source = CopySource(5 * self.max_size) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], self.split_source_to_part_defs(source, [self.max_size] * 5) ) def test_single_multipart_copy_remainder(self): self.assertGreaterEqual(self.min_size, 2) source = CopySource(5 * self.max_size + int(self.min_size / 2)) expected_part_count = 7 base_part_size = int(source.get_content_length() / expected_part_count) size_remainder = source.get_content_length() % expected_part_count expected_part_sizes = ( [base_part_size + 1] * size_remainder + [base_part_size] * (expected_part_count - size_remainder) ) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], self.split_source_to_part_defs(source, expected_part_sizes), ) def test_single_small_copy(self): source = CopySource(self.min_size - 1) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], [ # single small copy should be processed using `copy_file` # which does not have minimum file size limit part(source), ], ) def test_copy_then_small_copy(self): source_copy = CopySource(self.recommended_size) source_small_copy = CopySource(self.min_size - 1) write_intents = WriteIntent.wrap_sources_iterator([source_copy, source_small_copy]) self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy), part([source_small_copy]), # this means: download and then upload ], ) def test_small_copy_then_copy(self): self.assertGreater(self.min_size, MEGABYTE) source_small_copy = CopySource(self.min_size - MEGABYTE) source_copy = CopySource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator([source_small_copy, source_copy]) self.verify_emerge_plan_for_write_intents( write_intents, [ part([ source_small_copy, (source_copy, 0, MEGABYTE), ]), part(source_copy, MEGABYTE, self.recommended_size - MEGABYTE) ], ) def test_upload_small_copy_then_copy(self): source_upload = UploadSource(self.recommended_size) source_small_copy = CopySource(self.min_size - 1) source_copy = CopySource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator([source_upload, source_small_copy, source_copy]) self.verify_emerge_plan_for_write_intents( write_intents, [ part([ source_upload, source_small_copy, ]), part(source_copy), ] ) def test_upload_small_copy_x2_then_copy(self): source_upload = UploadSource(self.recommended_size) source_small_copy1 = CopySource(length=self.min_size - 1) source_small_copy2 = CopySource(length=self.min_size - 1) source_copy = CopySource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator( [source_upload, source_small_copy1, source_small_copy2, source_copy] ) self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_upload), part([ source_small_copy1, source_small_copy2, ]), part(source_copy), ], ) def test_upload_multiple_sources(self): self.assertEqual(self.recommended_size % 8, 0) unit_part_size = int(self.recommended_size / 8) uneven_part_size = 3 * unit_part_size sources = [ UploadSource(uneven_part_size) for i in range(8) ] self.verify_emerge_plan_for_write_intents( WriteIntent.wrap_sources_iterator(sources), [ part([ sources[0], sources[1], (sources[2], 0, 2 * unit_part_size), ]), part([ (sources[2], 2 * unit_part_size, unit_part_size), sources[3], sources[4], (sources[5], 0, unit_part_size), ]), part([ (sources[5], unit_part_size, 2 * unit_part_size), sources[6], sources[7], ]), ], ) def test_small_upload_not_enough_copy_then_upload(self): self.assertGreater(self.min_size, 2 * MEGABYTE) source_small_upload = UploadSource(self.min_size - 2 * MEGABYTE) source_copy = CopySource(self.min_size + MEGABYTE) source_upload = UploadSource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator( [source_small_upload, source_copy, source_upload] ) small_parts_len = source_small_upload.get_content_length() + source_copy.get_content_length() source_upload_split_offset = self.recommended_size - small_parts_len self.verify_emerge_plan_for_write_intents( write_intents, [ part([ source_small_upload, source_copy, (source_upload, 0, source_upload_split_offset), ]), part(source_upload, source_upload_split_offset, small_parts_len), ], ) def test_basic_local_overlap(self): source1 = UploadSource(self.recommended_size * 2) source2 = UploadSource(self.recommended_size * 2) write_intents = [ WriteIntent(source1), WriteIntent(source2, destination_offset=self.recommended_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [part(source1, 0, self.recommended_size)] + self.split_source_to_part_defs(source2, [self.recommended_size] * 2), ) def test_local_stairs_overlap(self): """ intent 0 #### intent 1 #### intent 2 #### intent 3 #### """ self.assertEqual(self.recommended_size % 4, 0) shift = int(self.recommended_size / 4) sources = [UploadSource(self.recommended_size) for i in range(4)] write_intents = [ WriteIntent(source, destination_offset=i * shift) for i, source in enumerate(sources) ] three_quarters = int(3 * self.recommended_size / 4) # 1234567 # su1: **** # su2: XXXX # su3: XXXX # su4: X*** self.verify_emerge_plan_for_write_intents( write_intents, [ part([ (sources[0], 0, three_quarters), (sources[-1], 0, shift) ]), part(sources[-1], shift, three_quarters), ], ) def test_local_remote_overlap_start(self): source_upload = UploadSource(self.recommended_size * 2) source_copy = CopySource(self.recommended_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy), ] self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy), part(source_upload, self.recommended_size, self.recommended_size), ], ) def test_local_remote_overlap_end(self): source_upload = UploadSource(self.recommended_size * 2) source_copy = CopySource(self.recommended_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy, destination_offset=self.recommended_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_upload, 0, self.recommended_size), part(source_copy), ], ) def test_local_remote_overlap_middle(self): source_upload = UploadSource(self.recommended_size * 3) source_copy = CopySource(self.recommended_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy, destination_offset=self.recommended_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_upload, 0, self.recommended_size), part(source_copy), part(source_upload, 2 * self.recommended_size, self.recommended_size), ], ) def test_local_small_copy_overlap(self): self.assertGreater(self.recommended_size, self.min_size * 3 - 3) source_upload = UploadSource(self.recommended_size) small_size = self.min_size - 1 source_copy1 = CopySource(small_size) source_copy2 = CopySource(small_size) source_copy3 = CopySource(small_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=small_size), WriteIntent(source_copy3, destination_offset=2 * small_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [part(source_upload)], ) def test_overlap_cause_small_copy_remainder_2_intent_case(self): self.assertGreater(self.min_size, 2 * MEGABYTE) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_copy1 = CopySource(copy_size) source_copy2 = CopySource(self.min_size) write_intents = [ WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), ] # 123456789 # sc1: ****** # sc2: XX*** self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, copy_size), part([ (source_copy2, 2 * MEGABYTE, self.min_size - 2 * MEGABYTE), # this means: download and then upload ]), ], ) def test_overlap_cause_small_copy_remainder_3_intent_case(self): self.assertGreater(self.min_size, MEGABYTE) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_copy1 = CopySource(copy_size) source_copy2 = CopySource(copy_size) source_copy3 = CopySource(copy_size) write_intents = [ WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), WriteIntent(source_copy3, destination_offset=2 * copy_overlap_offset), ] # 12345678901234 # sc1: *****X # sc2: X***** # sc3: XX**** self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, self.min_size), part(source_copy2, MEGABYTE, self.min_size), part([ (source_copy3, 2 * MEGABYTE, copy_overlap_offset), # this means: download and then upload ]), ], ) def test_overlap_protected_copy_and_upload(self): self.assertGreater(self.min_size, MEGABYTE) self.assertGreater(self.recommended_size, 2 * self.min_size) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_upload = UploadSource(self.recommended_size) source_copy1 = CopySource(copy_size) source_copy2 = CopySource(copy_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), ] upload_offset = copy_overlap_offset + copy_size # 123456789012 # su: XXXXXXXXXX**(...) # sc1: *****X # sc2: X***** self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, self.min_size), part(source_copy2, MEGABYTE, self.min_size), part(source_upload, upload_offset, self.recommended_size - upload_offset), ], ) def test_overlap_copy_and_small_copy_remainder_and_upload(self): self.assertGreater(self.min_size, 2 * MEGABYTE) self.assertGreater(self.recommended_size, self.min_size + MEGABYTE) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_upload = UploadSource(self.recommended_size) source_copy1 = CopySource(copy_size) source_copy2 = CopySource(self.min_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), ] # 12345678901 # su: XXXXXX*****(...) # sc1: ****** # sc2: XXXXX self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, copy_size), part(source_upload, copy_size, self.recommended_size - copy_size), ], ) def test_raise_on_hole(self): source_upload1 = UploadSource(self.recommended_size) source_upload2 = UploadSource(self.recommended_size) source_copy1 = CopySource(self.recommended_size) source_copy2 = CopySource(self.recommended_size) write_intents = [ WriteIntent(source_upload1), WriteIntent(source_upload2, destination_offset=self.recommended_size + 2 * MEGABYTE), WriteIntent(source_copy1, destination_offset=MEGABYTE), WriteIntent(source_copy2, destination_offset=self.recommended_size + 3 * MEGABYTE), ] hole_msg = ('Cannot emerge file with holes. ' 'Found hole range: ({}, {})'.format( write_intents[2].destination_end_offset, write_intents[1].destination_offset, )) with self.assertRaises(ValueError, hole_msg): self.planner.get_emerge_plan(write_intents) def test_empty_upload(self): source_upload = UploadSource(0) self.verify_emerge_plan_for_write_intents( [WriteIntent(source_upload)], [part(source_upload)], ) # yapf: enable def verify_emerge_plan_for_write_intents(self, write_intents, expected_part_defs): emerge_plan = self.planner.get_emerge_plan(write_intents) self.assert_same_part_definitions(emerge_plan, expected_part_defs) def split_source_to_part_defs(self, source, part_sizes): if isinstance(source, UploadSource): def_class = UploadEmergePartDefinition else: def_class = CopyEmergePartDefinition expected_part_defs = [] current_offset = 0 for part_size in part_sizes: expected_part_defs.append(def_class(source, current_offset, part_size)) current_offset += part_size return expected_part_defs def assert_same_part_definitions(self, emerge_plan, expected_part_defs): emerge_parts = list(emerge_plan.emerge_parts) self.assertEqual(len(emerge_parts), len(expected_part_defs)) for emerge_part, expected_part_def in zip(emerge_parts, expected_part_defs): emerge_part_def = emerge_part.part_definition self.assertIs(emerge_part_def.__class__, expected_part_def.__class__) if isinstance(emerge_part_def, UploadSubpartsEmergePartDefinition): upload_subparts = emerge_part_def.upload_subparts expected_subparts = expected_part_def.upload_subparts self.assertEqual(len(upload_subparts), len(expected_subparts)) for subpart, expected_subpart in zip(upload_subparts, expected_subparts): self.assertIs(subpart.__class__, expected_subpart.__class__) self.assertIs(subpart.outbound_source, expected_subpart.outbound_source) self.assertEqual(subpart.relative_offset, expected_subpart.relative_offset) self.assertEqual(subpart.length, expected_subpart.length) else: if isinstance(emerge_part_def, UploadEmergePartDefinition): self.assertIs(emerge_part_def.upload_source, expected_part_def.upload_source) else: self.assertIs(emerge_part_def.copy_source, expected_part_def.copy_source) self.assertEqual(emerge_part_def.relative_offset, expected_part_def.relative_offset) self.assertEqual(emerge_part_def.length, expected_part_def.length) b2-sdk-python-1.17.3/test/unit/replication/000077500000000000000000000000001426424117700204635ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/replication/conftest.py000066400000000000000000000037001426424117700226620ustar00rootroot00000000000000###################################################################### # # File: test/unit/replication/conftest.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from apiver_deps import B2Api, B2HttpApiConfig, Bucket, RawSimulator, ReplicationConfiguration, ReplicationMonitor, ReplicationRule, StubAccountInfo @pytest.fixture def api() -> B2Api: account_info = StubAccountInfo() api = B2Api( account_info, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator), ) simulator = api.session.raw_api account_id, master_key = simulator.create_account() api.authorize_account('production', account_id, master_key) # api_url = account_info.get_api_url() # account_auth_token = account_info.get_account_auth_token()1 return api @pytest.fixture def destination_bucket(api) -> Bucket: return api.create_bucket('destination-bucket', 'allPublic') @pytest.fixture def source_bucket(api, destination_bucket) -> Bucket: bucket = api.create_bucket('source-bucket', 'allPublic') bucket.replication = ReplicationConfiguration( rules=[ ReplicationRule( destination_bucket_id=destination_bucket.id_, name='name', file_name_prefix='folder/', # TODO: is last slash needed? ), ], source_key_id='hoho|trololo', ) return bucket @pytest.fixture def test_file(tmpdir) -> str: file = tmpdir.join('test.txt') file.write('whatever') return file @pytest.fixture def test_file_reversed(tmpdir) -> str: file = tmpdir.join('test-reversed.txt') file.write('revetahw') return file @pytest.fixture def monitor(source_bucket) -> ReplicationMonitor: return ReplicationMonitor( source_bucket, rule=source_bucket.replication.rules[0], ) b2-sdk-python-1.17.3/test/unit/replication/test_monitoring.py000066400000000000000000000156011426424117700242640ustar00rootroot00000000000000###################################################################### # # File: test/unit/replication/test_monitoring.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from apiver_deps import EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, FileRetentionSetting, ReplicationScanResult, RetentionMode, SSE_B2_AES SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_key', key_id='some-id'), ) RETENTION_GOVERNANCE = FileRetentionSetting(RetentionMode.GOVERNANCE, retain_until=1) DEFAULT_REPLICATION_RESULT = dict( source_replication_status=None, source_has_hide_marker=False, source_has_sse_c_enabled=False, source_has_large_metadata=False, source_has_file_retention=False, source_has_legal_hold=False, destination_replication_status=None, metadata_differs=None, hash_differs=None, ) def test_iter_pairs(source_bucket, destination_bucket, test_file, monitor): source_file = source_bucket.upload_local_file(test_file, 'folder/test.txt') source_subfolder_file = source_bucket.upload_local_file(test_file, 'folder/subfolder/test.txt') destination_subfolder_file = destination_bucket.upload_local_file( test_file, 'folder/subfolder/test.txt' ) destination_other_file = destination_bucket.upload_local_file( test_file, 'folder/subfolder/test2.txt' ) pairs = [ ( source_path and 'folder/' + source_path.relative_path, destination_path and 'folder/' + destination_path.relative_path, ) for source_path, destination_path in monitor.iter_pairs() ] assert set(pairs) == { (source_file.file_name, None), (source_subfolder_file.file_name, destination_subfolder_file.file_name), (None, destination_other_file.file_name), } def test_scan_source(source_bucket, test_file, monitor): # upload various types of files to source and get a report files = [ source_bucket.upload_local_file(test_file, 'folder/test-1.txt'), source_bucket.upload_local_file(test_file, 'folder/test-2.txt', encryption=SSE_B2_AES), source_bucket.upload_local_file(test_file, 'not-in-folder.txt'), # monitor should ignore this source_bucket.upload_local_file(test_file, 'folder/test-3.txt', encryption=SSE_C_AES), source_bucket.upload_local_file(test_file, 'folder/test-4.txt', encryption=SSE_C_AES), source_bucket.upload_local_file( test_file, 'folder/subfolder/test-5.txt', encryption=SSE_C_AES, file_retention=RETENTION_GOVERNANCE ), source_bucket.upload_local_file( test_file, 'folder/test-large-meta.txt', file_infos={ 'dummy-key': 'a' * 7000, }, ), source_bucket.upload_local_file( test_file, 'folder/test-large-meta-encrypted.txt', file_infos={ 'dummy-key': 'a' * 2048, }, encryption=SSE_C_AES, ), ] report = monitor.scan(scan_destination=False) assert report.counter_by_status[ReplicationScanResult(**DEFAULT_REPLICATION_RESULT)] == 2 assert report.counter_by_status[ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_has_sse_c_enabled': True, } )] == 2 assert report.counter_by_status[ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_has_sse_c_enabled': True, 'source_has_file_retention': True, } )] == 1 assert report.counter_by_status[ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_has_large_metadata': True, } )] == 1 assert report.counter_by_status[ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_has_sse_c_enabled': True, 'source_has_large_metadata': True, } )] == 1 # ---- first and last ---- assert report.samples_by_status_first[ReplicationScanResult(**DEFAULT_REPLICATION_RESULT,) ][0] == files[0] assert report.samples_by_status_last[ReplicationScanResult(**DEFAULT_REPLICATION_RESULT,) ][0] == files[1] def test_scan_source_and_destination( source_bucket, destination_bucket, test_file, test_file_reversed, monitor ): _ = [ # match source_bucket.upload_local_file(test_file, 'folder/test-1.txt'), destination_bucket.upload_local_file(test_file, 'folder/test-1.txt'), # missing on destination source_bucket.upload_local_file(test_file, 'folder/test-2.txt'), # missing on source destination_bucket.upload_local_file(test_file, 'folder/test-3.txt'), # metadata differs source_bucket.upload_local_file( test_file, 'folder/test-4.txt', file_infos={ 'haha': 'hoho', } ), destination_bucket.upload_local_file( test_file, 'folder/test-4.txt', file_infos={ 'hehe': 'hihi', } ), # hash differs source_bucket.upload_local_file(test_file, 'folder/test-5.txt'), destination_bucket.upload_local_file(test_file_reversed, 'folder/test-5.txt'), ] report = monitor.scan(scan_destination=True) # match assert report.counter_by_status[ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'metadata_differs': False, 'hash_differs': False, } )] == 1 # missing on destination assert report.counter_by_status[ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'destination_replication_status': None, } )] == 1 # missing on source assert report.counter_by_status[ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_replication_status': None, 'source_has_hide_marker': None, 'source_has_sse_c_enabled': None, 'source_has_large_metadata': None, 'source_has_file_retention': None, 'source_has_legal_hold': None, } )] == 1 # metadata differs assert report.counter_by_status[ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'metadata_differs': True, 'hash_differs': False, } )] == 1 # hash differs assert report.counter_by_status[ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'metadata_differs': False, 'hash_differs': True, } )] == 1 b2-sdk-python-1.17.3/test/unit/scan/000077500000000000000000000000001426424117700170765ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/scan/__init__.py000066400000000000000000000004451426424117700212120ustar00rootroot00000000000000###################################################################### # # File: test/unit/scan/__init__.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/scan/test_scan_policies.py000066400000000000000000000051011426424117700233170ustar00rootroot00000000000000###################################################################### # # File: test/unit/scan/test_scan_policies.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import re import pytest from apiver_deps import ScanPoliciesManager from apiver_deps_exception import InvalidArgument class TestScanPoliciesManager: def test_include_file_regexes_without_exclude(self): kwargs = {'include_file_regexes': '.*'} # valid regex with pytest.raises(InvalidArgument): ScanPoliciesManager(**kwargs) @pytest.mark.parametrize( 'param,exception', [ pytest.param( 'exclude_dir_regexes', InvalidArgument, marks=pytest.mark.apiver(from_ver=2) ), pytest.param( 'exclude_file_regexes', InvalidArgument, marks=pytest.mark.apiver(from_ver=2) ), pytest.param( 'include_file_regexes', InvalidArgument, marks=pytest.mark.apiver(from_ver=2) ), pytest.param('exclude_dir_regexes', re.error, marks=pytest.mark.apiver(to_ver=1)), pytest.param('exclude_file_regexes', re.error, marks=pytest.mark.apiver(to_ver=1)), pytest.param('include_file_regexes', re.error, marks=pytest.mark.apiver(to_ver=1)), ], ) def test_illegal_regex(self, param, exception): kwargs = { 'exclude_dir_regexes': '.*', 'exclude_file_regexes': '.*', 'include_file_regexes': '.*', param: '*', # invalid regex } with pytest.raises(exception): ScanPoliciesManager(**kwargs) @pytest.mark.parametrize( 'param,exception', [ pytest.param( 'exclude_modified_before', InvalidArgument, marks=pytest.mark.apiver(from_ver=2) ), pytest.param( 'exclude_modified_after', InvalidArgument, marks=pytest.mark.apiver(from_ver=2) ), pytest.param('exclude_modified_before', ValueError, marks=pytest.mark.apiver(to_ver=1)), pytest.param('exclude_modified_after', ValueError, marks=pytest.mark.apiver(to_ver=1)), ], ) def test_illegal_timestamp(self, param, exception): kwargs = { 'exclude_modified_before': 1, 'exclude_modified_after': 2, param: -1.0, # invalid range param } with pytest.raises(exception): ScanPoliciesManager(**kwargs) b2-sdk-python-1.17.3/test/unit/sync/000077500000000000000000000000001426424117700171265ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/sync/__init__.py000066400000000000000000000004451426424117700212420ustar00rootroot00000000000000###################################################################### # # File: test/unit/sync/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/sync/fixtures.py000066400000000000000000000027331426424117700213560ustar00rootroot00000000000000###################################################################### # # File: test/unit/sync/fixtures.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from apiver_deps import DEFAULT_SCAN_MANAGER, POLICY_MANAGER, CompareVersionMode, KeepOrDeleteMode, NewerFileSyncMode, Synchronizer @pytest.fixture(scope='session') def synchronizer_factory(): def get_synchronizer( policies_manager=DEFAULT_SCAN_MANAGER, dry_run=False, allow_empty_source=False, newer_file_mode=NewerFileSyncMode.RAISE_ERROR, keep_days_or_delete=KeepOrDeleteMode.NO_DELETE, keep_days=None, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=None, sync_policy_manager=POLICY_MANAGER, ): return Synchronizer( 1, policies_manager=policies_manager, dry_run=dry_run, allow_empty_source=allow_empty_source, newer_file_mode=newer_file_mode, keep_days_or_delete=keep_days_or_delete, keep_days=keep_days, compare_version_mode=compare_version_mode, compare_threshold=compare_threshold, sync_policy_manager=sync_policy_manager, ) return get_synchronizer @pytest.fixture def synchronizer(synchronizer_factory): return synchronizer_factory() b2-sdk-python-1.17.3/test/unit/sync/test_exception.py000066400000000000000000000043451426424117700225430ustar00rootroot00000000000000###################################################################### # # File: test/unit/sync/test_exception.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from apiver_deps_exception import ( EnvironmentEncodingError, InvalidArgument, IncompleteSync, UnsupportedFilename, check_invalid_argument, ) class TestSyncExceptions: def test_environment_encoding_error(self): try: raise EnvironmentEncodingError('fred', 'george') except EnvironmentEncodingError as e: assert str(e) == """file name fred cannot be decoded with system encoding (george). We think this is an environment error which you should workaround by setting your system encoding properly, for example like this: export LANG=en_US.UTF-8""", str(e) def test_invalid_argument(self): try: raise InvalidArgument('param', 'message') except InvalidArgument as e: assert str(e) == 'param message', str(e) def test_incomplete_sync(self): try: raise IncompleteSync() except IncompleteSync as e: assert str(e) == 'Incomplete sync: ', str(e) def test_unsupportedfilename_error(self): try: raise UnsupportedFilename('message', 'filename') except UnsupportedFilename as e: assert str(e) == 'message: filename', str(e) class TestCheckInvalidArgument: def test_custom_message(self): with pytest.raises(InvalidArgument): try: with check_invalid_argument('param', 'an error occurred', RuntimeError): raise RuntimeError() except InvalidArgument as exc: assert str(exc) == 'param an error occurred' raise def test_message_from_exception(self): with pytest.raises(InvalidArgument): try: with check_invalid_argument('param', '', RuntimeError): raise RuntimeError('an error occurred') except InvalidArgument as exc: assert str(exc) == 'param an error occurred' raise b2-sdk-python-1.17.3/test/unit/sync/test_sync.py000066400000000000000000001032531426424117700215170ustar00rootroot00000000000000###################################################################### # # File: test/unit/sync/test_sync.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from collections import defaultdict from unittest import mock from enum import Enum from functools import partial from apiver_deps import UpPolicy, B2DownloadAction, AbstractSyncEncryptionSettingsProvider, UploadSourceLocalFile, SyncPolicyManager from apiver_deps_exception import DestFileNewer, InvalidArgument from apiver_deps import KeepOrDeleteMode, NewerFileSyncMode, CompareVersionMode import pytest from ..fixtures.folder import * from .fixtures import * DAY = 86400000 # milliseconds TODAY = DAY * 100 # an arbitrary reference time for testing class IllegalEnum(Enum): ILLEGAL = 5100 class TestSynchronizer: @pytest.fixture(autouse=True) def setup(self, folder_factory, mocker, apiver): self.folder_factory = folder_factory self.local_folder_factory = partial(folder_factory, 'local') self.b2_folder_factory = partial(folder_factory, 'b2') self.reporter = mocker.MagicMock() self.apiver = apiver def _make_folder_sync_actions(self, synchronizer, *args, **kwargs): if self.apiver in ['v0', 'v1']: return synchronizer.make_folder_sync_actions(*args, **kwargs) return synchronizer._make_folder_sync_actions(*args, **kwargs) def assert_folder_sync_actions(self, synchronizer, src_folder, dst_folder, expected_actions): """ Checks the actions generated for one file. The file may or may not exist at the source, and may or may not exist at the destination. The source and destination files may have multiple versions. """ actions = list( self._make_folder_sync_actions( synchronizer, src_folder, dst_folder, TODAY, self.reporter, ) ) assert expected_actions == [str(a) for a in actions] @pytest.mark.apiver(to_ver=0) @pytest.mark.parametrize( 'args', [ { 'newer_file_mode': IllegalEnum.ILLEGAL }, { 'keep_days_or_delete': IllegalEnum.ILLEGAL }, ], ids=[ 'newer_file_mode', 'keep_days_or_delete', ] ) def test_illegal_args_up_to_v0(self, synchronizer_factory, apiver, args): from apiver_deps_exception import CommandError with pytest.raises(CommandError): synchronizer_factory(**args) @pytest.mark.apiver(from_ver=1) @pytest.mark.parametrize( 'args', [ { 'newer_file_mode': IllegalEnum.ILLEGAL }, { 'keep_days_or_delete': IllegalEnum.ILLEGAL }, ], ids=[ 'newer_file_mode', 'keep_days_or_delete', ] ) def test_illegal_args_up_v1_and_up(self, synchronizer_factory, apiver, args): with pytest.raises(InvalidArgument): synchronizer_factory(**args) def test_illegal(self, synchronizer): with pytest.raises(ValueError): src = self.local_folder_factory() dst = self.local_folder_factory() self.assert_folder_sync_actions(synchronizer, src, dst, []) # src: absent, dst: absent @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_empty(self, synchronizer, src_type, dst_type): src = self.folder_factory(src_type) dst = self.folder_factory(dst_type) self.assert_folder_sync_actions(synchronizer, src, dst, []) # # src: present, dst: absent @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 100)']), ('b2', 'local', ['b2_download(folder/a.txt, id_a_100, /dir/a.txt, 100)']), ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_100, folder/a.txt, 100)']), ], ) def test_not_there(self, synchronizer, src_type, dst_type, expected): src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,expected', [ ('local', ['b2_upload(/dir/directory/a.txt, folder/directory/a.txt, 100)']), ('b2', ['b2_copy(folder/directory/a.txt, id_d_100, folder/directory/a.txt, 100)']), ], ) def test_dir_not_there_b2_keepdays( self, synchronizer_factory, src_type, expected ): # reproduces issue 220 src = self.folder_factory(src_type, ('directory/a.txt', [100])) dst = self.b2_folder_factory() synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,expected', [ ('local', ['b2_upload(/dir/directory/a.txt, folder/directory/a.txt, 100)']), ('b2', ['b2_copy(folder/directory/a.txt, id_d_100, folder/directory/a.txt, 100)']), ], ) def test_dir_not_there_b2_delete( self, synchronizer_factory, src_type, expected ): # reproduces issue 220 src = self.folder_factory(src_type, ('directory/a.txt', [100])) dst = self.b2_folder_factory() synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) self.assert_folder_sync_actions(synchronizer, src, dst, expected) # # src: absent, dst: present @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_no_delete(self, synchronizer, src_type, dst_type): src = self.folder_factory(src_type) dst = self.folder_factory(dst_type, ('a.txt', [100])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), ('b2', 'local', ['local_delete(/dir/a.txt)']), ('b2', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), ], ) def test_delete(self, synchronizer_factory, src_type, dst_type, expected): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type) dst = self.folder_factory(dst_type, ('a.txt', [100])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), ('b2', 'local', ['local_delete(/dir/a.txt)']), ('b2', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), ], ) def test_delete_large(self, synchronizer_factory, src_type, dst_type, expected): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type) dst = self.folder_factory(dst_type, ('a.txt', [100], 10737418240)) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_delete_multiple_versions(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [100, 200])) expected = [ 'b2_delete(folder/a.txt, id_a_100, )', 'b2_delete(folder/a.txt, id_a_200, (old version))' ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_delete_hide_b2_multiple_versions(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) expected = [ 'b2_hide(folder/a.txt)', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_delete_hide_b2_multiple_versions_old(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY, TODAY - 5 * DAY])) expected = [ 'b2_hide(folder/a.txt)', 'b2_delete(folder/a.txt, id_a_8208000000, (old version))' ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep(self, synchronizer, src_type): src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep_days(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) expected = ['b2_delete(folder/a.txt, id_a_8294400000, (old version))'] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep_days_one_old( self, synchronizer_factory, src_type ): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=5 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory( ('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) ) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep_days_two_old( self, synchronizer_factory, src_type ): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory( ('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) ) expected = ['b2_delete(folder/a.txt, id_a_8121600000, (old version))'] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep_days_delete_hide_marker( self, synchronizer_factory, src_type ): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory( ('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) ) expected = [ 'b2_delete(folder/a.txt, id_a_8467200000, (hide marker))', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))', 'b2_delete(folder/a.txt, id_a_8121600000, (old version))' ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep_days_old_delete( self, synchronizer_factory, src_type ): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [-TODAY + 2 * DAY, TODAY - 4 * DAY])) expected = [ 'b2_delete(folder/a.txt, id_a_8467200000, (hide marker))', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_delete(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) expected = [ 'b2_delete(folder/a.txt, id_a_8640000000, (hide marker))', 'b2_delete(folder/a.txt, id_a_8467200000, (old version))', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))' ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) # # src same as dst @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_same(self, synchronizer, src_type, dst_type): src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type, ('a.txt', [100])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_same_leave_old_version(self, synchronizer, src_type): src = self.folder_factory(src_type, ('a.txt', [TODAY])) dst = self.b2_folder_factory(('a.txt', [TODAY, TODAY - 3 * DAY])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_same_clean_old_version(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type, ('a.txt', [TODAY - 3 * DAY])) dst = self.b2_folder_factory(('a.txt', [TODAY - 3 * DAY, TODAY - 4 * DAY])) expected = ['b2_delete(folder/a.txt, id_a_8294400000, (old version))'] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_keep_days_no_change_with_old_file(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type, ('a.txt', [TODAY - 3 * DAY])) dst = self.b2_folder_factory(('a.txt', [TODAY - 3 * DAY])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_same_delete_old_versions(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type, ('a.txt', [TODAY])) dst = self.b2_folder_factory(('a.txt', [TODAY, TODAY - 3 * DAY])) expected = ['b2_delete(folder/a.txt, id_a_8380800000, (old version))'] self.assert_folder_sync_actions(synchronizer, src, dst, expected) # # src newer than dst @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 200)']), ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)']), ], ) def test_never(self, synchronizer, src_type, dst_type, expected): src = self.folder_factory(src_type, ('a.txt', [200])) dst = self.folder_factory(dst_type, ('a.txt', [100])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,expected', [ ( 'local', [ 'b2_upload(/dir/a.txt, folder/a.txt, 8640000000)', 'b2_delete(folder/a.txt, id_a_8208000000, (old version))', ] ), ( 'b2', [ 'b2_copy(folder/a.txt, id_a_8640000000, folder/a.txt, 8640000000)', 'b2_delete(folder/a.txt, id_a_8208000000, (old version))', ] ), ], ) def test_newer_clean_old_versions(self, synchronizer_factory, src_type, expected): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2 ) src = self.folder_factory(src_type, ('a.txt', [TODAY])) dst = self.b2_folder_factory(('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY, TODAY - 5 * DAY])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,expected', [ ( 'local', [ 'b2_upload(/dir/a.txt, folder/a.txt, 8640000000)', 'b2_delete(folder/a.txt, id_a_8553600000, (old version))', 'b2_delete(folder/a.txt, id_a_8380800000, (old version))', ] ), ( 'b2', [ 'b2_copy(folder/a.txt, id_a_8640000000, folder/a.txt, 8640000000)', 'b2_delete(folder/a.txt, id_a_8553600000, (old version))', 'b2_delete(folder/a.txt, id_a_8380800000, (old version))', ] ), ], ) def test_newer_delete_old_versions(self, synchronizer_factory, src_type, expected): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type, ('a.txt', [TODAY])) dst = self.b2_folder_factory(('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) # # src older than dst @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 200)']), ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)']), ], ) def test_older(self, synchronizer, apiver, src_type, dst_type, expected): src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type, ('a.txt', [200])) with pytest.raises(DestFileNewer) as excinfo: self.assert_folder_sync_actions(synchronizer, src, dst, expected) messages = defaultdict( lambda: 'source file is older than destination: %s://a.txt with a time of 100 ' 'cannot be synced to %s://a.txt with a time of 200, ' 'unless a valid newer_file_mode is provided', v0='source file is older than destination: %s://a.txt with a time of 100 ' 'cannot be synced to %s://a.txt with a time of 200, ' 'unless --skipNewer or --replaceNewer is provided' ) # yapf: disable assert str(excinfo.value) == messages[apiver] % (src_type, dst_type) @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_older_skip(self, synchronizer_factory, src_type, dst_type): synchronizer = synchronizer_factory(newer_file_mode=NewerFileSyncMode.SKIP) src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type, ('a.txt', [200])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 100)']), ('b2', 'local', ['b2_download(folder/a.txt, id_a_100, /dir/a.txt, 100)']), ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_100, folder/a.txt, 100)']), ], ) def test_older_replace(self, synchronizer_factory, src_type, dst_type, expected): synchronizer = synchronizer_factory(newer_file_mode=NewerFileSyncMode.REPLACE) src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type, ('a.txt', [200])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,expected', [ ( 'local', [ 'b2_upload(/dir/a.txt, folder/a.txt, 100)', 'b2_delete(folder/a.txt, id_a_200, (old version))', ] ), ( 'b2', [ 'b2_copy(folder/a.txt, id_a_100, folder/a.txt, 100)', 'b2_delete(folder/a.txt, id_a_200, (old version))', ] ), ], ) def test_older_replace_delete(self, synchronizer_factory, src_type, expected): synchronizer = synchronizer_factory( newer_file_mode=NewerFileSyncMode.REPLACE, keep_days_or_delete=KeepOrDeleteMode.DELETE ) src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.b2_folder_factory(('a.txt', [200])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) # # compareVersions option @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_compare_none_newer(self, synchronizer_factory, src_type, dst_type): synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.NONE) src = self.folder_factory(src_type, ('a.txt', [200])) dst = self.folder_factory(dst_type, ('a.txt', [100])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_compare_none_older(self, synchronizer_factory, src_type, dst_type): synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.NONE) src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type, ('a.txt', [200])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_compare_size_equal(self, synchronizer_factory, src_type, dst_type): synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.SIZE) src = self.folder_factory(src_type, ('a.txt', [200], 10)) dst = self.folder_factory(dst_type, ('a.txt', [100], 10)) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 200)']), ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)']), ], ) def test_compare_size_not_equal(self, synchronizer_factory, src_type, dst_type, expected): synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.SIZE) src = self.folder_factory(src_type, ('a.txt', [200], 11)) dst = self.folder_factory(dst_type, ('a.txt', [100], 10)) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ( 'local', 'b2', [ 'b2_upload(/dir/a.txt, folder/a.txt, 200)', 'b2_delete(folder/a.txt, id_a_100, (old version))' ] ), ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), ( 'b2', 'b2', [ 'b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)', 'b2_delete(folder/a.txt, id_a_100, (old version))' ] ), ], ) def test_compare_size_not_equal_delete( self, synchronizer_factory, src_type, dst_type, expected ): synchronizer = synchronizer_factory( compare_version_mode=CompareVersionMode.SIZE, keep_days_or_delete=KeepOrDeleteMode.DELETE ) src = self.folder_factory(src_type, ('a.txt', [200], 11)) dst = self.folder_factory(dst_type, ('a.txt', [100], 10)) self.assert_folder_sync_actions(synchronizer, src, dst, expected) # FIXME: rewrite this test to not use mock.call checks when all of Synchronizers tests are rewritten to test_bucket # style - i.e. with simulated api and fake files returned from methods. Then, checking EncryptionSetting used for # transmission will be done by the underlying simulator. def test_encryption_b2_to_local(self, synchronizer_factory, apiver): local = self.local_folder_factory() remote = self.b2_folder_factory(('directory/b.txt', [100])) synchronizer = synchronizer_factory() encryption = object() bucket = mock.MagicMock() provider = TstEncryptionSettingsProvider(encryption, encryption) download_action = next( iter( self._make_folder_sync_actions( synchronizer, remote, local, TODAY, self.reporter, encryption_settings_provider=provider ) ) ) with mock.patch.object(B2DownloadAction, '_ensure_directory_existence'): try: download_action.do_action(bucket, self.reporter) except: pass assert bucket.mock_calls[0] == mock.call.download_file_by_id( 'id_d_100', progress_listener=mock.ANY, encryption=encryption ) if apiver in ['v0', 'v1']: file_version_kwarg = 'file_version_info' else: file_version_kwarg = 'file_version' provider.get_setting_for_download.assert_has_calls( [mock.call( bucket=bucket, **{file_version_kwarg: mock.ANY}, )] ) # FIXME: rewrite this test to not use mock.call checks when all of Synchronizers tests are rewritten to test_bucket # style - i.e. with simulated api and fake files returned from methods. Then, checking EncryptionSetting used for # transmission will be done by the underlying simulator. def test_encryption_local_to_b2(self, synchronizer_factory): local = self.local_folder_factory(('directory/a.txt', [100])) remote = self.b2_folder_factory() synchronizer = synchronizer_factory() encryption = object() bucket = mock.MagicMock() provider = TstEncryptionSettingsProvider(encryption, encryption) upload_action = next( iter( self._make_folder_sync_actions( synchronizer, local, remote, TODAY, self.reporter, encryption_settings_provider=provider ) ) ) with mock.patch.object(UploadSourceLocalFile, 'check_path_and_get_size'): try: upload_action.do_action(bucket, self.reporter) except: pass assert bucket.mock_calls == [ mock.call.upload( mock.ANY, 'folder/directory/a.txt', file_info={'src_last_modified_millis': '100'}, progress_listener=mock.ANY, encryption=encryption ) ] assert provider.get_setting_for_upload.mock_calls == [ mock.call( bucket=bucket, b2_file_name='folder/directory/a.txt', file_info={'src_last_modified_millis': '100'}, length=10 ) ] # FIXME: rewrite this test to not use mock.call checks when all of Synchronizers tests are rewritten to test_bucket # style - i.e. with simulated api and fake files returned from methods. Then, checking EncryptionSetting used for # transmission will be done by the underlying simulator. def test_encryption_b2_to_b2(self, synchronizer_factory, apiver): src = self.b2_folder_factory(('directory/a.txt', [100])) dst = self.b2_folder_factory() synchronizer = synchronizer_factory() source_encryption = object() destination_encryption = object() bucket = mock.MagicMock() provider = TstEncryptionSettingsProvider(source_encryption, destination_encryption) copy_action = next( iter( self._make_folder_sync_actions( synchronizer, src, dst, TODAY, self.reporter, encryption_settings_provider=provider ) ) ) copy_action.do_action(bucket, self.reporter) assert bucket.mock_calls == [ mock.call.copy( 'id_d_100', 'folder/directory/a.txt', length=10, source_content_type='text/plain', source_file_info={'in_b2': 'yes'}, progress_listener=mock.ANY, source_encryption=source_encryption, destination_encryption=destination_encryption ) ] if apiver in ['v0', 'v1']: file_version_kwarg = 'source_file_version_info' additional_kwargs = {'target_file_info': None} else: file_version_kwarg = 'source_file_version' additional_kwargs = {} assert provider.get_source_setting_for_copy.mock_calls == [ mock.call( bucket=mock.ANY, **{file_version_kwarg: mock.ANY}, ) ] assert provider.get_destination_setting_for_copy.mock_calls == [ mock.call( bucket=mock.ANY, dest_b2_file_name='folder/directory/a.txt', **additional_kwargs, **{file_version_kwarg: mock.ANY}, ) ] def test_custom_sync_manager_policy(self, synchronizer_factory): class MySyncPolicyManager(SyncPolicyManager): def get_policy_class(self, sync_type, delete, keep_days): return UpPolicy synchronizer = synchronizer_factory( compare_version_mode=CompareVersionMode.SIZE, keep_days_or_delete=KeepOrDeleteMode.DELETE, sync_policy_manager=MySyncPolicyManager(), ) src = self.folder_factory('local', ('a.txt', [200], 11)) dst = self.folder_factory('b2', ('a.txt', [100], 10)) # normally_expected = [ # 'b2_upload(/dir/a.txt, folder/a.txt, 200)', # 'b2_delete(folder/a.txt, id_a_100, (old version))' # ] expected = ['b2_upload(/dir/a.txt, folder/a.txt, 200)'] self.assert_folder_sync_actions(synchronizer, src, dst, expected) class TstEncryptionSettingsProvider(AbstractSyncEncryptionSettingsProvider): def __init__(self, source_encryption_setting, destination_encryption_setting): self.get_setting_for_upload = mock.MagicMock( side_effect=lambda *a, **kw: destination_encryption_setting ) self.get_source_setting_for_copy = mock.MagicMock( side_effect=lambda *a, **kw: source_encryption_setting ) self.get_destination_setting_for_copy = mock.MagicMock( side_effect=lambda *a, **kw: destination_encryption_setting ) self.get_setting_for_download = mock.MagicMock( side_effect=lambda *a, **kw: source_encryption_setting ) def get_setting_for_upload(self, *a, **kw): """overwritten in __init__""" def get_source_setting_for_copy(self, *a, **kw): """overwritten in __init__""" def get_destination_setting_for_copy(self, *a, **kw): """overwritten in __init__""" def get_setting_for_download(self, *a, **kw): """overwritten in __init__""" b2-sdk-python-1.17.3/test/unit/sync/test_sync_report.py000066400000000000000000000031611426424117700231070ustar00rootroot00000000000000###################################################################### # # File: test/unit/sync/test_sync_report.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from unittest.mock import MagicMock import pytest from apiver_deps import SyncReport class TestSyncReport: def test_bad_terminal(self): stdout = MagicMock() stdout.write = MagicMock( side_effect=[ UnicodeEncodeError('codec', u'foo', 100, 105, 'artificial UnicodeEncodeError') ] + list(range(25)) ) sync_report = SyncReport(stdout, False) sync_report.print_completion('transferred: 123.txt') @pytest.mark.apiver(to_ver=1) def test_legacy_methods(self): stdout = MagicMock() sync_report = SyncReport(stdout, False) assert not sync_report.total_done assert not sync_report.local_done assert 0 == sync_report.total_count assert 0 == sync_report.local_file_count sync_report.local_done = True assert sync_report.local_done assert sync_report.total_done sync_report.local_file_count = 8 assert 8 == sync_report.local_file_count assert 8 == sync_report.total_count sync_report.update_local(7) assert 15 == sync_report.total_count assert 15 == sync_report.local_file_count sync_report = SyncReport(stdout, False) assert not sync_report.total_done sync_report.end_local() assert sync_report.total_done b2-sdk-python-1.17.3/test/unit/test_base.py000066400000000000000000000035101426424117700204740ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_base.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import re import unittest from contextlib import contextmanager from typing import List, Optional import apiver_deps from apiver_deps import B2Api from b2sdk.v2 import FullApplicationKey class TestBase(unittest.TestCase): @contextmanager def assertRaises(self, exc, msg=None): try: yield except exc as e: if msg is not None: if msg != str(e): assert False, "expected message '%s', but got '%s'" % (msg, str(e)) else: assert False, 'should have thrown %s' % (exc,) @contextmanager def assertRaisesRegexp(self, expected_exception, expected_regexp): try: yield except expected_exception as e: if not re.search(expected_regexp, str(e)): assert False, "expected message '%s', but got '%s'" % (expected_regexp, str(e)) else: assert False, 'should have thrown %s' % (expected_exception,) def create_key( api: B2Api, capabilities: List[str], key_name: str, valid_duration_seconds: Optional[int] = None, bucket_id: Optional[str] = None, name_prefix: Optional[str] = None, ) -> FullApplicationKey: """apiver-agnostic B2Api.create_key""" result = api.create_key( capabilities=capabilities, key_name=key_name, valid_duration_seconds=valid_duration_seconds, bucket_id=bucket_id, name_prefix=name_prefix, ) if apiver_deps.V <= 1: return FullApplicationKey.from_create_response(result) return result b2-sdk-python-1.17.3/test/unit/test_exception.py000066400000000000000000000135771426424117700215760ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_exception.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from b2sdk.exception import ResourceNotFound from apiver_deps_exception import ( AlreadyFailed, B2Error, BadJson, BadUploadUrl, BucketIdNotFound, CapExceeded, Conflict, DuplicateBucketName, FileAlreadyHidden, FileNotPresent, interpret_b2_error, InvalidAuthToken, MissingPart, PartSha1Mismatch, ServiceError, StorageCapExceeded, TooManyRequests, TransactionCapExceeded, Unauthorized, UnknownError, ) class TestB2Error: def test_plain_ascii(self): assert 'message' == str(B2Error('message')) def test_unicode(self): assert '\u81ea\u7531' == str(B2Error('\u81ea\u7531')) class TestExceptions: def test_bad_upload_url_exception(self): try: raise BadUploadUrl('foo') except BadUploadUrl as e: assert not e.should_retry_http() assert e.should_retry_upload() assert str(e) == 'Bad upload url: foo', str(e) def test_already_failed_exception(self): try: raise AlreadyFailed('foo') except AlreadyFailed as e: assert str(e) == 'Already failed: foo', str(e) @pytest.mark.apiver(to_ver=1) def test_command_error(self): from apiver_deps_exception import CommandError try: raise CommandError('foo') except CommandError as e: assert str(e) == 'foo', str(e) class TestInterpretError: def test_file_already_hidden(self): self._check_one(FileAlreadyHidden, 400, 'already_hidden', '', {}) assert 'File already hidden: file.txt' == \ str(interpret_b2_error(400, 'already_hidden', '', {}, {'fileName': 'file.txt'})) def test_bad_json(self): self._check_one(BadJson, 400, 'bad_json', '', {}) def test_file_not_present(self): self._check_one(FileNotPresent, 400, 'no_such_file', '', {}) self._check_one(FileNotPresent, 400, 'file_not_present', '', {}) self._check_one(FileNotPresent, 404, 'not_found', '', {}) assert 'File not present: file.txt' == \ str(interpret_b2_error(404, 'not_found', '', {}, {'fileName': 'file.txt'})) assert 'File not present: 01010101' == \ str(interpret_b2_error(404, 'not_found', '', {}, {'fileId': '01010101'})) def test_file_or_bucket_not_present(self): self._check_one(ResourceNotFound, 404, None, None, {}) assert 'No such file, bucket, or endpoint: ' == \ str(interpret_b2_error(404, None, None, {})) def test_duplicate_bucket_name(self): self._check_one(DuplicateBucketName, 400, 'duplicate_bucket_name', '', {}) assert 'Bucket name is already in use: my-bucket' == \ str( interpret_b2_error( 400, 'duplicate_bucket_name', '', {}, {'bucketName': 'my-bucket'} ) ) def test_missing_part(self): self._check_one(MissingPart, 400, 'missing_part', '', {}) assert 'Part number has not been uploaded: my-file-id' == \ str(interpret_b2_error(400, 'missing_part', '', {}, {'fileId': 'my-file-id'})) def test_part_sha1_mismatch(self): self._check_one(PartSha1Mismatch, 400, 'part_sha1_mismatch', '', {}) assert 'Part number my-file-id has wrong SHA1' == \ str(interpret_b2_error(400, 'part_sha1_mismatch', '', {}, {'fileId': 'my-file-id'})) def test_unauthorized(self): self._check_one(Unauthorized, 401, '', '', {}) def test_invalid_auth_token(self): self._check_one(InvalidAuthToken, 401, 'bad_auth_token', '', {}) self._check_one(InvalidAuthToken, 401, 'expired_auth_token', '', {}) def test_storage_cap_exceeded(self): self._check_one((CapExceeded, StorageCapExceeded), 403, 'storage_cap_exceeded', '', {}) def test_transaction_cap_exceeded(self): self._check_one( (CapExceeded, TransactionCapExceeded), 403, 'transaction_cap_exceeded', '', {} ) def test_conflict(self): self._check_one(Conflict, 409, '', '', {}) def test_too_many_requests_with_retry_after_header(self): retry_after = 200 error = self._check_one( TooManyRequests, 429, '', '', {'retry-after': retry_after}, ) assert error.retry_after_seconds == retry_after def test_too_many_requests_without_retry_after_header(self): error = self._check_one(TooManyRequests, 429, '', '', {}) assert error.retry_after_seconds is None @pytest.mark.apiver( from_ver=3 ) # previous apivers throw this as well, but BucketIdNotFound is a different class in them def test_bad_bucket_id(self): error = self._check_one( BucketIdNotFound, 400, 'bad_bucket_id', '', {}, {'bucketId': '1001'} ) assert error.bucket_id == '1001' def test_service_error(self): error = interpret_b2_error(500, 'code', 'message', {}) assert isinstance(error, ServiceError) assert '500 code message' == str(error) def test_unknown_error(self): error = interpret_b2_error(499, 'code', 'message', {}) assert isinstance(error, UnknownError) assert 'Unknown error: 499 code message' == str(error) @classmethod def _check_one( cls, expected_class, status, code, message, response_headers, post_params=None, ): actual_exception = interpret_b2_error(status, code, message, response_headers, post_params) assert isinstance(actual_exception, expected_class) return actual_exception b2-sdk-python-1.17.3/test/unit/test_included_modules.py000066400000000000000000000011601426424117700231000ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_included_modules.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pathlib import pytest from b2sdk import requests from b2sdk.requests.included_source_meta import included_source_meta def test_requests_notice_file(): with (pathlib.Path(requests.__file__).parent / 'NOTICE').open('r') as notice_file: assert notice_file.read() == included_source_meta.files['NOTICE'] b2-sdk-python-1.17.3/test/unit/test_session.py000066400000000000000000000046051426424117700212530ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_session.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from unittest import mock from .account_info.fixtures import * # noqa from .fixtures import * # noqa class TestAuthorizeAccount: @pytest.fixture(autouse=True) def setup(self, b2_session): self.b2_session = b2_session @pytest.mark.parametrize( 'authorize_call_kwargs', [ pytest.param( dict( account_id=mock.ANY, auth_token=mock.ANY, api_url=mock.ANY, download_url=mock.ANY, recommended_part_size=mock.ANY, absolute_minimum_part_size=mock.ANY, application_key='456', realm='dev', s3_api_url=mock.ANY, allowed=mock.ANY, application_key_id='123', ), marks=pytest.mark.apiver(from_ver=2) ), pytest.param( dict( account_id=mock.ANY, auth_token=mock.ANY, api_url=mock.ANY, download_url=mock.ANY, minimum_part_size=mock.ANY, application_key='456', realm='dev', s3_api_url=mock.ANY, allowed=mock.ANY, application_key_id='123', ), marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_simple_authorization(self, authorize_call_kwargs): self.b2_session.authorize_account('dev', '123', '456') self.b2_session.raw_api.authorize_account.assert_called_once_with( 'http://api.backblazeb2.xyz:8180', '123', '456' ) assert self.b2_session.cache.clear.called is False self.b2_session.account_info.set_auth_data.assert_called_once_with(**authorize_call_kwargs) def test_clear_cache(self): self.b2_session.account_info.is_same_account.return_value = False self.b2_session.authorize_account('dev', '123', '456') assert self.b2_session.cache.clear.called is True b2-sdk-python-1.17.3/test/unit/v0/000077500000000000000000000000001426424117700164775ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/v0/__init__.py000066400000000000000000000004431426424117700206110ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/v0/apiver/000077500000000000000000000000001426424117700177655ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/v0/apiver/__init__.py000066400000000000000000000006221426424117700220760ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/apiver/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details b2-sdk-python-1.17.3/test/unit/v0/apiver/apiver_deps.py000066400000000000000000000005241426424117700226410ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/apiver/apiver_deps.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.v0 import * # noqa V = 0 b2-sdk-python-1.17.3/test/unit/v0/apiver/apiver_deps_exception.py000066400000000000000000000005411426424117700247160ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/apiver/apiver_deps_exception.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.v0.exception import * # noqa b2-sdk-python-1.17.3/test/unit/v0/deps.py000066400000000000000000000011311426424117700200000ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/deps.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # TODO: This module is used in old-style unit tests, written separately for v0 and v1. # It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details from apiver_deps import * b2-sdk-python-1.17.3/test/unit/v0/deps_exception.py000066400000000000000000000011551426424117700220640ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/deps_exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # TODO: This module is used in old-style unit tests, written separately for v0 and v1. # It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details from apiver_deps_exception import * b2-sdk-python-1.17.3/test/unit/v0/test_bounded_queue_executor.py000066400000000000000000000037551426424117700246640ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_bounded_queue_executor.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import concurrent.futures as futures import time from .deps import BoundedQueueExecutor from ..test_base import TestBase class TestBoundedQueueExecutor(TestBase): def setUp(self): unbounded_executor = futures.ThreadPoolExecutor(max_workers=1) self.executor = BoundedQueueExecutor(unbounded_executor, 1) def tearDown(self): self.executor.shutdown() def test_return_future(self): future_1 = self.executor.submit(lambda: 1) print(future_1) self.assertEqual(1, future_1.result()) def test_blocking(self): # This doesn't actually test that it waits, but it does exercise the code. # Make some futures using a function that takes a little time. def sleep_and_return_fcn(n): def fcn(): time.sleep(0.01) return n return fcn futures = [self.executor.submit(sleep_and_return_fcn(i)) for i in range(10)] # Check the answers answers = list(map(lambda f: f.result(), futures)) self.assertEqual(list(range(10)), answers) def test_no_exceptions(self): f = self.executor.submit(lambda: 1) self.executor.shutdown() self.assertEqual(0, self.executor.get_num_exceptions()) self.assertTrue(f.exception() is None) def test_two_exceptions(self): def thrower(): raise Exception('test_exception') f1 = self.executor.submit(thrower) f2 = self.executor.submit(thrower) self.executor.shutdown() self.assertEqual(2, self.executor.get_num_exceptions()) self.assertFalse(f1.exception() is None) self.assertEqual('test_exception', str(f2.exception())) b2-sdk-python-1.17.3/test/unit/v0/test_bucket.py000066400000000000000000001407631426424117700214000ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_bucket.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from io import BytesIO import os import platform import unittest.mock as mock import pytest from ..test_base import TestBase from .deps_exception import ( AlreadyFailed, B2Error, InvalidAuthToken, InvalidMetadataDirective, InvalidRange, InvalidUploadSource, MaxRetriesExceeded, UnsatisfiableRange, SSECKeyError, ) from .deps import B2Api from .deps import LargeFileUploadState from .deps import DownloadDestBytes, PreSeekedDownloadDest from .deps import FileVersionInfo from .deps import LegalHold, FileRetentionSetting, RetentionMode, NO_RETENTION_FILE_SETTING from .deps import MetadataDirectiveMode from .deps import Part from .deps import AbstractProgressListener from .deps import StubAccountInfo, RawSimulator, BucketSimulator, FakeResponse, FileSimulator from .deps import ParallelDownloader from .deps import SimpleDownloader from .deps import UploadSourceBytes from .deps import hex_sha1_of_bytes, TempDir from .deps import EncryptionAlgorithm, EncryptionSetting, EncryptionSettingFactory, EncryptionMode, \ EncryptionKey, SSE_NONE, SSE_B2_AES from .deps import CopySource, UploadSourceLocalFile, WriteIntent SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_key', key_id='some-id'), ) SSE_C_AES_NO_SECRET = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=None, key_id='some-id'), ) SSE_C_AES_2 = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_other_key', key_id='some-id-2'), ) SSE_C_AES_2_NO_SECRET = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=None, key_id='some-id-2'), ) SSE_C_AES_FROM_SERVER = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(key_id=None, secret=None), ) def write_file(path, data): with open(path, 'wb') as f: f.write(data) class StubProgressListener(AbstractProgressListener): """ Implementation of a progress listener that remembers what calls were made, and returns them as a short string to use in unit tests. For a total byte count of 100, and updates at 33 and 66, the returned string looks like: "100: 33 66" """ def __init__(self): self.total = None self.history = [] self.last_byte_count = 0 def get_history(self): return ' '.join(self.history) def set_total_bytes(self, total_byte_count): assert total_byte_count is not None assert self.total is None, 'set_total_bytes called twice' self.total = total_byte_count assert len(self.history) == 0, self.history self.history.append('%d:' % (total_byte_count,)) def bytes_completed(self, byte_count): self.last_byte_count = byte_count self.history.append(str(byte_count)) def is_valid(self, **kwargs): valid, _ = self.is_valid_reason(**kwargs) return valid def is_valid_reason( self, check_closed=True, check_progress=True, check_monotonic_progress=False ): progress_end = -1 if self.history[progress_end] == 'closed': progress_end = -2 # self.total != self.last_byte_count may be a consequence of non-monotonic # progress, so we want to check this first if check_monotonic_progress: prev = 0 for val in map(int, self.history[1:progress_end]): if val < prev: return False, 'non-monotonic progress' prev = val if self.total != self.last_byte_count: return False, 'total different than last_byte_count' if check_closed and self.history[-1] != 'closed': return False, 'no "closed" at the end of history' if check_progress and len(self.history[1:progress_end]) < 2: return False, 'progress in history has less than 2 entries' return True, '' def close(self): self.history.append('closed') class CanRetry(B2Error): """ An exception that can be retryable, or not. """ def __init__(self, can_retry): super(CanRetry, self).__init__(None, None, None, None, None) self.can_retry = can_retry def should_retry_upload(self): return self.can_retry class TestCaseWithBucket(TestBase): RAW_SIMULATOR_CLASS = RawSimulator def setUp(self): self.bucket_name = 'my-bucket' self.simulator = self.RAW_SIMULATOR_CLASS() self.account_info = StubAccountInfo() self.api = B2Api(self.account_info, raw_api=self.simulator) (self.account_id, self.master_key) = self.simulator.create_account() self.api.authorize_account('production', self.account_id, self.master_key) self.api_url = self.account_info.get_api_url() self.account_auth_token = self.account_info.get_account_auth_token() self.bucket = self.api.create_bucket('my-bucket', 'allPublic') self.bucket_id = self.bucket.id_ def assertBucketContents(self, expected, *args, **kwargs): """ *args and **kwargs are passed to self.bucket.ls() """ actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket.ls(*args, **kwargs) ] self.assertEqual(expected, actual) def _make_data(self, approximate_length): """ Generate a sequence of bytes to use in testing an upload. Don't repeat a short pattern, so we're sure that the different parts of a large file are actually different. Returns bytes. """ fragments = [] so_far = 0 while so_far < approximate_length: fragment = ('%d:' % so_far).encode('utf-8') so_far += len(fragment) fragments.append(fragment) return b''.join(fragments) class TestReauthorization(TestCaseWithBucket): def testCreateBucket(self): class InvalidAuthTokenWrapper: def __init__(self, original_function): self.__original_function = original_function self.__name__ = original_function.__name__ self.__called = False def __call__(self, *args, **kwargs): if self.__called: return self.__original_function(*args, **kwargs) self.__called = True raise InvalidAuthToken('message', 401) self.simulator.create_bucket = InvalidAuthTokenWrapper(self.simulator.create_bucket) self.bucket = self.api.create_bucket('your-bucket', 'allPublic') class TestListParts(TestCaseWithBucket): def testEmpty(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) self.assertEqual([], list(self.bucket.list_parts(file1.file_id, batch_size=1))) def testThree(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) content = b'hello world' content_sha1 = hex_sha1_of_bytes(content) large_file_upload_state = mock.MagicMock() large_file_upload_state.has_error.return_value = False self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 1, large_file_upload_state ).result() self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 2, large_file_upload_state ).result() self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 3, large_file_upload_state ).result() expected_parts = [ Part('9999', 1, 11, content_sha1), Part('9999', 2, 11, content_sha1), Part('9999', 3, 11, content_sha1), ] self.assertEqual(expected_parts, list(self.bucket.list_parts(file1.file_id, batch_size=1))) class TestUploadPart(TestCaseWithBucket): def test_error_in_state(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) content = b'hello world' file_progress_listener = mock.MagicMock() large_file_upload_state = LargeFileUploadState(file_progress_listener) large_file_upload_state.set_error('test error') try: self.bucket.api.services.upload_manager.upload_part( self.bucket.id_, file1.file_id, UploadSourceBytes(content), 1, large_file_upload_state ).result() self.fail('should have thrown') except AlreadyFailed: pass class TestListUnfinished(TestCaseWithBucket): def test_empty(self): self.assertEqual([], list(self.bucket.list_unfinished_large_files())) def test_one(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) self.assertEqual([file1], list(self.bucket.list_unfinished_large_files())) def test_three(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) file2 = self.bucket.start_large_file('file2.txt', 'text/plain', {}) file3 = self.bucket.start_large_file('file3.txt', 'text/plain', {}) self.assertEqual( [file1, file2, file3], list(self.bucket.list_unfinished_large_files(batch_size=1)) ) def _make_file(self, file_id, file_name): return self.bucket.start_large_file(file_name, 'text/plain', {}) class TestGetFileInfo(TestCaseWithBucket): def test_version_by_name(self): data = b'hello world' a_id = self.bucket.upload_bytes(data, 'a').id_ info = self.bucket.get_file_info_by_name('a') self.assertIsInstance(info, FileVersionInfo) expected = ( a_id, 'a', 11, 'upload', 'b2/x-auto', 'none', NO_RETENTION_FILE_SETTING, LegalHold.UNSET ) actual = ( info.id_, info.file_name, info.size, info.action, info.content_type, info.server_side_encryption.mode.value, info.file_retention, info.legal_hold, ) self.assertEqual(expected, actual) def test_version_by_name_file_lock(self): bucket = self.api.create_bucket( 'my-bucket-with-file-lock', 'allPublic', is_file_lock_enabled=True ) data = b'hello world' legal_hold = LegalHold.ON file_retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 100) bucket.upload_bytes(data, 'a', file_retention=file_retention, legal_hold=legal_hold) file_version = bucket.get_file_info_by_name('a') actual = (file_version.legal_hold, file_version.file_retention) self.assertEqual((legal_hold, file_retention), actual) low_perm_account_info = StubAccountInfo() low_perm_api = B2Api(low_perm_account_info, raw_api=self.simulator) low_perm_key_resp = self.api.create_key( key_name='lowperm', capabilities=[ 'listKeys', 'listBuckets', 'listFiles', 'readFiles', ] ) low_perm_api.authorize_account( 'production', low_perm_key_resp['applicationKeyId'], low_perm_key_resp['applicationKey'] ) low_perm_bucket = low_perm_api.get_bucket_by_name('my-bucket-with-file-lock') file_version = low_perm_bucket.get_file_info_by_name('a') actual = (file_version.legal_hold, file_version.file_retention) expected = (LegalHold.UNKNOWN, FileRetentionSetting(RetentionMode.UNKNOWN)) self.assertEqual(expected, actual) def test_version_by_id(self): data = b'hello world' b_id = self.bucket.upload_bytes(data, 'b').id_ info = self.bucket.get_file_info_by_id(b_id) self.assertIsInstance(info, FileVersionInfo) expected = (b_id, 'b', 11, 'upload', 'b2/x-auto') actual = (info.id_, info.file_name, info.size, info.action, info.content_type) self.assertEqual(expected, actual) class TestLs(TestCaseWithBucket): def test_empty(self): self.assertEqual([], list(self.bucket.ls('foo'))) def test_one_file_at_root(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '') def test_three_files_at_root(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb') self.bucket.upload_bytes(data, 'ccc') expected = [ ('a', 11, 'upload', None), ('bb', 11, 'upload', None), ('ccc', 11, 'upload', None) ] self.assertBucketContents(expected, '') def test_three_files_in_dir(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb/1') self.bucket.upload_bytes(data, 'bb/2/sub1') self.bucket.upload_bytes(data, 'bb/2/sub2') self.bucket.upload_bytes(data, 'bb/3') self.bucket.upload_bytes(data, 'ccc') expected = [ ('bb/1', 11, 'upload', None), ('bb/2/sub1', 11, 'upload', 'bb/2/'), ('bb/3', 11, 'upload', None) ] self.assertBucketContents(expected, 'bb', fetch_count=1) def test_three_files_multiple_versions(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb/1') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/3') self.bucket.upload_bytes(data, 'ccc') expected = [ ('9998', 'bb/1', 11, 'upload', None), ('9995', 'bb/2', 11, 'upload', None), ('9996', 'bb/2', 11, 'upload', None), ('9997', 'bb/2', 11, 'upload', None), ('9994', 'bb/3', 11, 'upload', None), ] actual = [ (info.id_, info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket.ls('bb', show_versions=True, fetch_count=1) ] self.assertEqual(expected, actual) def test_started_large_file(self): self.bucket.start_large_file('hello.txt') expected = [('hello.txt', 0, 'start', None)] self.assertBucketContents(expected, '', show_versions=True) def test_hidden_file(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') self.bucket.hide_file('hello.txt') expected = [('hello.txt', 0, 'hide', None), ('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_delete_file_version(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') files = self.bucket.list_file_names('hello.txt', 1)['files'] file_dict = files[0] file_id = file_dict['fileId'] data = b'hello new world' self.bucket.upload_bytes(data, 'hello.txt') self.bucket.delete_file_version(file_id, 'hello.txt') expected = [('hello.txt', 15, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) class TestListVersions(TestCaseWithBucket): def test_encryption(self): data = b'hello world' a = self.bucket.upload_bytes(data, 'a') a_id = a.id_ self.assertEqual(a.server_side_encryption, SSE_NONE) b = self.bucket.upload_bytes(data, 'b', encryption=SSE_B2_AES) self.assertEqual(b.server_side_encryption, SSE_B2_AES) b_id = b.id_ # c_id = self.bucket.upload_bytes(data, 'c', encryption=SSE_NONE).id_ # TODO self.bucket.copy(a_id, 'd', destination_encryption=SSE_B2_AES) self.bucket.copy( b_id, 'e', destination_encryption=SSE_C_AES, file_info={}, content_type='text/plain' ) actual = [info for info in self.bucket.list_file_versions('a')['files']][0] actual = EncryptionSettingFactory.from_file_version_dict(actual) self.assertEqual(SSE_NONE, actual) # bucket default actual = [info for info in self.bucket.list_file_versions('b')['files']][0] actual = EncryptionSettingFactory.from_file_version_dict(actual) self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 # actual = [info for info in self.bucket.list_file_versions('c')][0] # actual = EncryptionSettingFactory.from_file_version_dict(actual) # self.assertEqual(SSE_NONE, actual) # explicitly requested none actual = [info for info in self.bucket.list_file_versions('d')['files']][0] actual = EncryptionSettingFactory.from_file_version_dict(actual) self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 actual = [info for info in self.bucket.list_file_versions('e')['files']][0] actual = EncryptionSettingFactory.from_file_version_dict(actual) self.assertEqual(SSE_C_AES_NO_SECRET, actual) # explicitly requested sse-c class TestCopyFile(TestCaseWithBucket): def test_copy_without_optional_params(self): file_id = self._make_file() self.bucket.copy_file(file_id, 'hello_new.txt') expected = [('hello.txt', 11, 'upload', None), ('hello_new.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_range(self): file_id = self._make_file() self.bucket.copy_file(file_id, 'hello_new.txt', bytes_range=(3, 9)) expected = [('hello.txt', 11, 'upload', None), ('hello_new.txt', 7, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_invalid_metadata(self): file_id = self._make_file() try: self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.COPY, content_type='application/octet-stream', ) self.fail('should have raised InvalidMetadataDirective') except InvalidMetadataDirective as e: self.assertEqual( 'content_type and file_info should be None when metadata_directive is COPY', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_invalid_metadata_replace(self): file_id = self._make_file() try: self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.REPLACE, ) self.fail('should have raised InvalidMetadataDirective') except InvalidMetadataDirective as e: self.assertEqual( 'content_type cannot be None when metadata_directive is REPLACE', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_replace_metadata(self): file_id = self._make_file() self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.REPLACE, content_type='text/plain', ) expected = [ ('hello.txt', 11, 'upload', 'b2/x-auto', None), ('hello_new.txt', 11, 'upload', 'text/plain', None), ] actual = [ (info.file_name, info.size, info.action, info.content_type, folder) for (info, folder) in self.bucket.ls(show_versions=True) ] self.assertEqual(expected, actual) def test_copy_with_unsatisfied_range(self): file_id = self._make_file() try: self.bucket.copy_file( file_id, 'hello_new.txt', bytes_range=(12, 15), ) self.fail('should have raised UnsatisfiableRange') except UnsatisfiableRange as e: self.assertEqual( 'The range in the request is outside the size of the file', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_different_bucket(self): source_bucket = self.api.create_bucket('source-bucket', 'allPublic') file_id = self._make_file(source_bucket) self.bucket.copy_file(file_id, 'hello_new.txt') def ls(bucket): return [ (info.file_name, info.size, info.action, folder) for (info, folder) in bucket.ls(show_versions=True) ] expected = [('hello.txt', 11, 'upload', None)] self.assertEqual(expected, ls(source_bucket)) expected = [('hello_new.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_retention(self): for data in [self._make_data(self.simulator.MIN_PART_SIZE * 3), b'hello']: for length in [None, len(data)]: with self.subTest(real_length=len(data), length=length): file_id = self.bucket.upload_bytes(data, 'original_file').id_ resulting_file_version = self.bucket.copy( file_id, 'copied_file', file_retention=FileRetentionSetting(RetentionMode.COMPLIANCE, 100), legal_hold=LegalHold.ON, max_part_size=400, ) self.assertEqual( FileRetentionSetting(RetentionMode.COMPLIANCE, 100), resulting_file_version.file_retention ) self.assertEqual(LegalHold.ON, resulting_file_version.legal_hold) def test_copy_encryption(self): data = b'hello_world' a = self.bucket.upload_bytes(data, 'a') a_id = a.id_ self.assertEqual(a.server_side_encryption, SSE_NONE) b = self.bucket.upload_bytes(data, 'b', encryption=SSE_B2_AES) self.assertEqual(b.server_side_encryption, SSE_B2_AES) b_id = b.id_ c = self.bucket.upload_bytes(data, 'c', encryption=SSE_C_AES) self.assertEqual(c.server_side_encryption, SSE_C_AES_NO_SECRET) c_id = c.id_ for length in [None, len(data)]: for kwargs, expected_encryption in [ (dict(file_id=a_id, destination_encryption=SSE_B2_AES), SSE_B2_AES), ( dict( file_id=a_id, destination_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain' ), SSE_C_AES_NO_SECRET ), ( dict( file_id=a_id, destination_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain' ), SSE_C_AES_NO_SECRET ), (dict(file_id=b_id), SSE_NONE), (dict(file_id=b_id, source_encryption=SSE_B2_AES), SSE_NONE), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_B2_AES ), SSE_B2_AES ), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain' ), SSE_C_AES_NO_SECRET ), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain' ), SSE_C_AES_NO_SECRET ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain' ), SSE_NONE ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain' ), SSE_NONE ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES ), SSE_C_AES_NO_SECRET ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_B2_AES, source_file_info={'old': 'value'}, source_content_type='text/plain' ), SSE_B2_AES ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_B2_AES, file_info={'new': 'value'}, content_type='text/plain' ), SSE_B2_AES ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES_2, source_file_info={'old': 'value'}, source_content_type='text/plain' ), SSE_C_AES_2_NO_SECRET ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES_2, file_info={'new': 'value'}, content_type='text/plain' ), SSE_C_AES_2_NO_SECRET ), ]: with self.subTest(kwargs=kwargs, length=length, data=data): file_info = self.bucket.copy(**kwargs, new_file_name='new_file', length=length) self.assertTrue(isinstance(file_info, FileVersionInfo)) self.assertEqual(file_info.server_side_encryption, expected_encryption) def _make_file(self, bucket=None): data = b'hello world' actual_bucket = bucket or self.bucket return actual_bucket.upload_bytes(data, 'hello.txt').id_ class TestUpload(TestCaseWithBucket): def test_upload_bytes(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1') self.assertTrue(isinstance(file_info, FileVersionInfo)) self._check_file_contents('file1', data) def test_upload_bytes_file_retention(self): data = b'hello world' retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 150) file_info = self.bucket.upload_bytes( data, 'file1', file_retention=retention, legal_hold=LegalHold.ON ) self._check_file_contents('file1', data) self.assertEqual(retention, file_info.file_retention) self.assertEqual(LegalHold.ON, file_info.legal_hold) def test_upload_bytes_sse_b2(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1', encryption=SSE_B2_AES) self.assertTrue(isinstance(file_info, FileVersionInfo)) self.assertEqual(file_info.server_side_encryption, SSE_B2_AES) def test_upload_bytes_sse_c(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1', encryption=SSE_C_AES) self.assertTrue(isinstance(file_info, FileVersionInfo)) self.assertEqual(SSE_C_AES_NO_SECRET, file_info.server_side_encryption) def test_upload_local_file_sse_b2(self): with TempDir() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1', encryption=SSE_B2_AES) self.assertTrue(isinstance(file_info, FileVersionInfo)) self.assertEqual(file_info.server_side_encryption, SSE_B2_AES) self._check_file_contents('file1', data) def test_upload_local_file_sse_c(self): with TempDir() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1', encryption=SSE_C_AES) self.assertTrue(isinstance(file_info, FileVersionInfo)) self.assertEqual(SSE_C_AES_NO_SECRET, file_info.server_side_encryption) self._check_file_contents('file1', data) def test_upload_local_file_retention(self): with TempDir() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 150) file_info = self.bucket.upload_local_file( path, 'file1', encryption=SSE_C_AES, file_retention=retention, legal_hold=LegalHold.ON ) self._check_file_contents('file1', data) self.assertEqual(retention, file_info.file_retention) self.assertEqual(LegalHold.ON, file_info.legal_hold) def test_upload_bytes_progress(self): data = b'hello world' progress_listener = StubProgressListener() self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertTrue(progress_listener.is_valid()) def test_upload_local_file(self): with TempDir() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) self.bucket.upload_local_file(path, 'file1') self._check_file_contents('file1', data) @pytest.mark.skipif(platform.system() == 'Windows', reason='no os.mkfifo() on Windows') def test_upload_fifo(self): with TempDir() as d: path = os.path.join(d, 'file1') os.mkfifo(path) with self.assertRaises(InvalidUploadSource): self.bucket.upload_local_file(path, 'file1') @pytest.mark.skipif(platform.system() == 'Windows', reason='no os.symlink() on Windows') def test_upload_dead_symlink(self): with TempDir() as d: path = os.path.join(d, 'file1') os.symlink('non-existing', path) with self.assertRaises(InvalidUploadSource): self.bucket.upload_local_file(path, 'file1') def test_upload_one_retryable_error(self): self.simulator.set_upload_errors([CanRetry(True)]) data = b'hello world' self.bucket.upload_bytes(data, 'file1') def test_upload_file_one_fatal_error(self): self.simulator.set_upload_errors([CanRetry(False)]) data = b'hello world' with self.assertRaises(CanRetry): self.bucket.upload_bytes(data, 'file1') def test_upload_file_too_many_retryable_errors(self): self.simulator.set_upload_errors([CanRetry(True)] * 6) data = b'hello world' with self.assertRaises(MaxRetriesExceeded): self.bucket.upload_bytes(data, 'file1') def test_upload_large(self): data = self._make_data(self.simulator.MIN_PART_SIZE * 3) progress_listener = StubProgressListener() self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_no_parts(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertNotEqual(large_file_id, file_info.id_) # it's not a match if there are no parts self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_all_parts_there(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[:part_size]) self._upload_part(large_file_id, 2, data[part_size:2 * part_size]) self._upload_part(large_file_id, 3, data[2 * part_size:]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_part_does_not_match(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 3, data[:part_size]) # wrong part number for this data progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_wrong_part_size(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[:part_size + 1]) # one byte to much progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_file_info(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1', {'property': 'value1'}) self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes( data, 'file1', progress_listener=progress_listener, file_infos={'property': 'value1'} ) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_file_info_does_not_match(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1', {'property': 'value1'}) self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes( data, 'file1', progress_listener=progress_listener, file_infos={'property': 'value2'} ) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def _start_large_file(self, file_name, file_info=None): if file_info is None: file_info = {} large_file_info = self.simulator.start_large_file( self.api_url, self.account_auth_token, self.bucket_id, file_name, None, file_info ) return large_file_info['fileId'] def _upload_part(self, large_file_id, part_number, part_data): part_stream = BytesIO(part_data) upload_info = self.simulator.get_upload_part_url( self.api_url, self.account_auth_token, large_file_id ) self.simulator.upload_part( upload_info['uploadUrl'], upload_info['authorizationToken'], part_number, len(part_data), hex_sha1_of_bytes(part_data), part_stream ) def _check_file_contents(self, file_name, expected_contents): download = DownloadDestBytes() with FileSimulator.dont_check_encryption(): self.bucket.download_file_by_name(file_name, download) self.assertEqual(expected_contents, download.get_bytes_written()) class TestConcatenate(TestCaseWithBucket): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.concatenate(sources, file_name=file_name, encryption=encryption) def test_create_remote(self): data = b'hello world' f1_id = self.bucket.upload_bytes(data, 'f1').id_ f2_id = self.bucket.upload_bytes(data, 'f1').id_ with TempDir() as d: path = os.path.join(d, 'file') write_file(path, data) created_file = self._create_remote( [ CopySource(f1_id, length=len(data), offset=0), UploadSourceLocalFile(path), CopySource(f2_id, length=len(data), offset=0), ], file_name='created_file' ) self.assertIsInstance(created_file, FileVersionInfo) actual = ( created_file.id_, created_file.file_name, created_file.size, created_file.server_side_encryption ) expected = ('9997', 'created_file', 33, SSE_NONE) self.assertEqual(expected, actual) def test_create_remote_encryption(self): for data in [b'hello_world', self._make_data(self.simulator.MIN_PART_SIZE * 3)]: f1_id = self.bucket.upload_bytes(data, 'f1', encryption=SSE_C_AES).id_ f2_id = self.bucket.upload_bytes(data, 'f1', encryption=SSE_C_AES_2).id_ with TempDir() as d: path = os.path.join(d, 'file') write_file(path, data) created_file = self._create_remote( [ CopySource(f1_id, length=len(data), offset=0, encryption=SSE_C_AES), UploadSourceLocalFile(path), CopySource(f2_id, length=len(data), offset=0, encryption=SSE_C_AES_2), ], file_name='created_file_%s' % (len(data),), encryption=SSE_C_AES ) self.assertIsInstance(created_file, FileVersionInfo) actual = ( created_file.id_, created_file.file_name, created_file.size, created_file.server_side_encryption ) expected = ( mock.ANY, 'created_file_%s' % (len(data),), mock.ANY, # FIXME: this should be equal to len(data) * 3, # but there is a problem in the simulator/test code somewhere SSE_C_AES_NO_SECRET ) self.assertEqual(expected, actual) class TestCreateFile(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.create_file( [wi for wi in WriteIntent.wrap_sources_iterator(sources)], file_name=file_name, encryption=encryption ) class TestConcatenateStream(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.concatenate_stream(sources, file_name=file_name, encryption=encryption) class TestCreateFileStream(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.create_file_stream( [wi for wi in WriteIntent.wrap_sources_iterator(sources)], file_name=file_name, encryption=encryption ) # Downloads class DownloadTests: DATA = 'abcdefghijklmnopqrs' def setUp(self): super(DownloadTests, self).setUp() self.file_info = self.bucket.upload_bytes(self.DATA.encode(), 'file1') self.encrypted_file_info = self.bucket.upload_bytes( self.DATA.encode(), 'enc_file1', encryption=SSE_C_AES ) self.download_dest = DownloadDestBytes() self.progress_listener = StubProgressListener() def _verify(self, expected_result, check_progress_listener=True): assert self.download_dest.get_bytes_written() == expected_result.encode() if check_progress_listener: valid, reason = self.progress_listener.is_valid_reason( check_closed=False, check_progress=False, check_monotonic_progress=True, ) assert valid, reason def test_download_by_id_no_progress(self): self.bucket.download_file_by_id(self.file_info.id_, self.download_dest) self._verify(self.DATA, check_progress_listener=False) def test_download_by_name_no_progress(self): self.bucket.download_file_by_name('file1', self.download_dest) self._verify(self.DATA, check_progress_listener=False) def test_download_by_name_progress(self): self.bucket.download_file_by_name('file1', self.download_dest, self.progress_listener) self._verify(self.DATA) def test_download_by_id_progress(self): self.bucket.download_file_by_id( self.file_info.id_, self.download_dest, self.progress_listener ) self._verify(self.DATA) def test_download_by_id_progress_partial(self): self.bucket.download_file_by_id( self.file_info.id_, self.download_dest, self.progress_listener, range_=(3, 9) ) self._verify('defghij') def test_download_by_id_progress_exact_range(self): self.bucket.download_file_by_id( self.file_info.id_, self.download_dest, self.progress_listener, range_=(0, 18) ) self._verify(self.DATA) def test_download_by_id_progress_range_one_off(self): with self.assertRaises( InvalidRange, msg='A range of 0-19 was requested (size of 20), but cloud could only serve 19 of that', ): self.bucket.download_file_by_id( self.file_info.id_, self.download_dest, self.progress_listener, range_=(0, 19), ) def test_download_by_id_progress_partial_inplace_overwrite(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # ||||||| # vvvvvvv # # 123defghij1234567890 with TempDir() as d: path = os.path.join(d, 'file2') download_dest = PreSeekedDownloadDest(seek_target=3, local_file_path=path) data = b'12345678901234567890' write_file(path, data) self.bucket.download_file_by_id( self.file_info.id_, download_dest, self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'123defghij1234567890') def test_download_by_id_progress_partial_shifted_overwrite(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # ||||||| # vvvvvvv # # 1234567defghij567890 with TempDir() as d: path = os.path.join(d, 'file2') download_dest = PreSeekedDownloadDest(seek_target=7, local_file_path=path) data = b'12345678901234567890' write_file(path, data) self.bucket.download_file_by_id( self.file_info.id_, download_dest, self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'1234567defghij567890') def test_download_by_id_no_progress_encryption(self): self.bucket.download_file_by_id( self.encrypted_file_info.id_, self.download_dest, encryption=SSE_C_AES ) self._verify(self.DATA, check_progress_listener=False) def test_download_by_id_no_progress_wrong_encryption(self): with self.assertRaises(SSECKeyError): self.bucket.download_file_by_id( self.encrypted_file_info.id_, self.download_dest, encryption=SSE_C_AES_2 ) def _check_local_file_contents(self, path, expected_contents): with open(path, 'rb') as f: contents = f.read() self.assertEqual(contents, expected_contents) # download empty file class EmptyFileDownloadScenarioMixin: """ use with DownloadTests, but not for TestDownloadParallel as it does not like empty files """ def test_download_by_name_empty_file(self): self.file_info = self.bucket.upload_bytes(b'', 'empty') self.bucket.download_file_by_name('empty', self.download_dest, self.progress_listener) self._verify('') # actual tests class TestDownloadDefault(DownloadTests, EmptyFileDownloadScenarioMixin, TestCaseWithBucket): pass class TestDownloadSimple(DownloadTests, EmptyFileDownloadScenarioMixin, TestCaseWithBucket): def setUp(self): super(TestDownloadSimple, self).setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [SimpleDownloader(force_chunk_size=20)] class TestDownloadParallel(DownloadTests, TestCaseWithBucket): def setUp(self): super(TestDownloadParallel, self).setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [ ParallelDownloader( force_chunk_size=2, max_streams=999, min_part_size=2, ) ] # Truncated downloads class TruncatedFakeResponse(FakeResponse): """ A special FakeResponse class which returns only the first 4 bytes of data. Use it to test followup retries for truncated download issues. """ def __init__(self, *args, **kwargs): super(TruncatedFakeResponse, self).__init__(*args, **kwargs) self.data_bytes = self.data_bytes[:4] class TruncatedDownloadBucketSimulator(BucketSimulator): RESPONSE_CLASS = TruncatedFakeResponse class TruncatedDownloadRawSimulator(RawSimulator): BUCKET_SIMULATOR_CLASS = TruncatedDownloadBucketSimulator class TestCaseWithTruncatedDownloadBucket(TestCaseWithBucket): RAW_SIMULATOR_CLASS = TruncatedDownloadRawSimulator ####### actual tests of truncated downloads class TestTruncatedDownloadSimple(DownloadTests, TestCaseWithTruncatedDownloadBucket): def setUp(self): super(TestTruncatedDownloadSimple, self).setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [SimpleDownloader(force_chunk_size=20)] class TestTruncatedDownloadParallel(DownloadTests, TestCaseWithTruncatedDownloadBucket): def setUp(self): super(TestTruncatedDownloadParallel, self).setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [ ParallelDownloader( force_chunk_size=3, max_streams=2, min_part_size=2, ) ] b2-sdk-python-1.17.3/test/unit/v0/test_copy_manager.py000066400000000000000000000140061426424117700225550ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_copy_manager.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from ..test_base import TestBase from .deps_exception import SSECKeyIdMismatchInCopy from .deps import MetadataDirectiveMode from .deps import EncryptionAlgorithm, EncryptionSetting, EncryptionMode, EncryptionKey, SSE_B2_AES from b2sdk.transfer.outbound.copy_manager import CopyManager from b2sdk.http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_key', key_id='some-id'), ) SSE_C_AES_2 = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_other_key', key_id='some-id-2'), ) class TestCopyManager(TestBase): def test_establish_sse_c_replace(self): file_info = {'some_key': 'some_value'} content_type = 'text/plain' metadata_directive, new_file_info, new_content_type = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.REPLACE, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info=None, source_content_type=None, ) self.assertEqual( ( MetadataDirectiveMode.REPLACE, { 'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id' }, content_type ), (metadata_directive, new_file_info, new_content_type) ) def test_establish_sse_c_copy_no_enc(self): file_info = {} content_type = 'text/plain' metadata_directive, new_file_info, new_content_type = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=None, source_server_side_encryption=None, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, {}, content_type), (metadata_directive, new_file_info, new_content_type) ) def test_establish_sse_c_copy_b2(self): file_info = {} content_type = 'text/plain' metadata_directive, new_file_info, new_content_type = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_B2_AES, source_server_side_encryption=None, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, {}, content_type), (metadata_directive, new_file_info, new_content_type) ) def test_establish_sse_c_copy_same_key_id(self): file_info = None content_type = 'text/plain' metadata_directive, new_file_info, new_content_type = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, None, content_type), (metadata_directive, new_file_info, new_content_type) ) def test_establish_sse_c_copy_sources_given(self): metadata_directive, new_file_info, new_content_type = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=None, destination_content_type=None, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info={ 'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id-2' }, source_content_type='text/plain', ) self.assertEqual( ( MetadataDirectiveMode.REPLACE, { 'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id' }, 'text/plain' ), (metadata_directive, new_file_info, new_content_type) ) def test_establish_sse_c_copy_sources_unknown(self): for source_file_info, source_content_type in [ (None, None), ({ 'a': 'b' }, None), (None, 'text/plain'), ]: with self.subTest( source_file_info=source_file_info, source_content_type=source_content_type ): with self.assertRaises( SSECKeyIdMismatchInCopy, 'attempting to copy file using MetadataDirectiveMode.COPY without providing source_file_info ' 'and source_content_type for differing sse_c_key_ids: source="some-id-2", destination="some-id"' ): CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=None, destination_content_type=None, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info=source_file_info, source_content_type=source_content_type, ) b2-sdk-python-1.17.3/test/unit/v0/test_download_dest.py000066400000000000000000000070711426424117700227430ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_download_dest.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os from ..test_base import TestBase from .deps import DownloadDestLocalFile, DownloadDestProgressWrapper, PreSeekedDownloadDest from .deps import ProgressListenerForTest from .deps import TempDir class TestDownloadDestLocalFile(TestBase): expected_result = 'hello world' def _make_dest(self, temp_dir): file_path = os.path.join(temp_dir, "test.txt") return DownloadDestLocalFile(file_path), file_path def test_write_and_set_mod_time(self): """ Check that the file gets written and that its mod time gets set. """ mod_time = 1500222333000 with TempDir() as temp_dir: download_dest, file_path = self._make_dest(temp_dir) with download_dest.make_file_context( "file_id", "file_name", 100, "content_type", "sha1", {}, mod_time ) as f: f.write(b'hello world') with open(file_path, 'rb') as f: self.assertEqual( self.expected_result.encode(), f.read(), ) self.assertEqual(mod_time, int(os.path.getmtime(file_path) * 1000)) def test_failed_write_deletes_partial_file(self): with TempDir() as temp_dir: download_dest, file_path = self._make_dest(temp_dir) try: with download_dest.make_file_context( "file_id", "file_name", 100, "content_type", "sha1", {}, 1500222333000 ) as f: f.write(b'hello world') raise Exception('test error') except Exception as e: self.assertEqual('test error', str(e)) self.assertFalse(os.path.exists(file_path), msg='failed download should be deleted') class TestPreSeekedDownloadDest(TestDownloadDestLocalFile): expected_result = '123hello world567890' def _make_dest(self, temp_dir): file_path = os.path.join(temp_dir, "test.txt") with open(file_path, 'wb') as f: f.write(b'12345678901234567890') return PreSeekedDownloadDest(local_file_path=file_path, seek_target=3), file_path class TestDownloadDestProgressWrapper(TestBase): def test_write_and_set_mod_time_and_progress(self): """ Check that the file gets written and that its mod time gets set. """ mod_time = 1500222333000 with TempDir() as temp_dir: file_path = os.path.join(temp_dir, "test.txt") download_local_file = DownloadDestLocalFile(file_path) progress_listener = ProgressListenerForTest() download_dest = DownloadDestProgressWrapper(download_local_file, progress_listener) with download_dest.make_file_context( "file_id", "file_name", 100, "content_type", "sha1", {}, mod_time ) as f: f.write(b'hello world\n') with open(file_path, 'rb') as f: self.assertEqual(b'hello world\n', f.read()) self.assertEqual(mod_time, int(os.path.getmtime(file_path) * 1000)) self.assertEqual( [ 'set_total_bytes(100)', 'bytes_completed(12)', 'close()', ], progress_listener.get_calls(), ) b2-sdk-python-1.17.3/test/unit/v0/test_file_metadata.py000066400000000000000000000030301426424117700226630ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_file_metadata.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from .deps import FileMetadata from ..test_base import TestBase def snake_to_camel(name): camel = ''.join(s.title() for s in name.split('_')) return camel[:1].lower() + camel[1:] class TestFileMetadata(TestBase): KWARGS = { 'file_id': '4_deadbeaf3b3e38a957f100d1e_f1042665d79618ae7_d20200903_m194254_c000_v0001053_t0048', 'file_name': 'foo.txt', 'content_type': 'text/plain', 'content_length': '1', 'content_sha1': '4518012e1b365e504001dbc94120624f15b8bbd5', 'file_info': {}, } # yapf: disable INFO_DICT = {snake_to_camel(k): v for k, v in KWARGS.items()} def test_verified_sha1(self): metadata = FileMetadata(**self.KWARGS) self.assertTrue(metadata.content_sha1_verified) self.assertEqual(metadata.as_info_dict(), self.INFO_DICT) def test_unverified_sha1(self): kwargs = self.KWARGS.copy() kwargs['content_sha1'] = 'unverified:' + kwargs['content_sha1'] info_dict = self.INFO_DICT.copy() info_dict['contentSha1'] = 'unverified:' + info_dict['contentSha1'] metadata = FileMetadata(**kwargs) self.assertFalse(metadata.content_sha1_verified) self.assertEqual(metadata.as_info_dict(), info_dict) b2-sdk-python-1.17.3/test/unit/v0/test_policy.py000066400000000000000000000063621426424117700214160ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_policy.py # # Copyright 2019, Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from unittest.mock import MagicMock from ..test_base import TestBase from .deps import FileVersionInfo from .deps import LocalSyncPath, B2SyncPath from .deps import B2Folder from .deps import make_b2_keep_days_actions class TestMakeB2KeepDaysActions(TestBase): def setUp(self): self.keep_days = 7 self.today = 100 * 86400 self.one_day_millis = 86400 * 1000 def test_no_versions(self): self.check_one_answer(True, [], []) def test_new_version_no_action(self): self.check_one_answer(True, [(1, -5, 'upload')], []) def test_no_source_one_old_version_hides(self): # An upload that is old gets deleted if there is no source file. self.check_one_answer(False, [(1, -10, 'upload')], ['b2_hide(folder/a)']) def test_old_hide_causes_delete(self): # A hide marker that is old gets deleted, as do the things after it. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'hide'), (3, -20, 'upload')], ['b2_delete(folder/a, 2, (hide marker))', 'b2_delete(folder/a, 3, (old version))'] ) def test_old_upload_causes_delete(self): # An upload that is old stays if there is a source file, but things # behind it go away. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'upload'), (3, -20, 'upload')], ['b2_delete(folder/a, 3, (old version))'] ) def test_out_of_order_dates(self): # The one at date -3 will get deleted because the one before it is old. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'upload'), (3, -3, 'upload')], ['b2_delete(folder/a, 3, (old version))'] ) def check_one_answer(self, has_source, id_relative_date_action_list, expected_actions): source_file = LocalSyncPath('a', 'a', 100, 10) if has_source else None dest_file_versions = [ FileVersionInfo( id_=id_, file_name='folder/' + 'a', upload_timestamp=self.today + relative_date * self.one_day_millis, action=action, size=100, file_info={}, content_type='text/plain', content_sha1='content_sha1', ) for (id_, relative_date, action) in id_relative_date_action_list ] dest_file = B2SyncPath( 'a', selected_version=dest_file_versions[0], all_versions=dest_file_versions ) if dest_file_versions else None bucket = MagicMock() api = MagicMock() api.get_bucket_by_name.return_value = bucket dest_folder = B2Folder('bucket-1', 'folder', api) actual_actions = list( make_b2_keep_days_actions( source_file, dest_file, dest_folder, dest_folder, self.keep_days, self.today ) ) actual_action_strs = [str(a) for a in actual_actions] self.assertEqual(expected_actions, actual_action_strs) b2-sdk-python-1.17.3/test/unit/v0/test_progress.py000066400000000000000000000037671426424117700217710ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_progress.py # # Copyright 2019, Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from io import BytesIO from ..test_base import TestBase from .deps import StreamWithHash from .deps import hex_sha1_of_bytes class TestHashingStream(TestBase): def setUp(self): self.data = b'01234567' self.stream = StreamWithHash(BytesIO(self.data)) self.hash = hex_sha1_of_bytes(self.data) self.expected = self.data + self.hash.encode() def test_no_argument(self): output = self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_less(self): output = self.stream.read(len(self.data) - 1) self.assertEqual(len(output), len(self.data) - 1) output += self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_equal(self): output = self.stream.read(len(self.data)) self.assertEqual(len(output), len(self.data)) output += self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_more(self): output = self.stream.read(len(self.data) + 1) self.assertEqual(len(output), len(self.data) + 1) output += self.stream.read() self.assertEqual(self.expected, output) def test_one_by_one(self): for expected_byte in self.expected: self.assertEqual(bytes((expected_byte,)), self.stream.read(1)) self.assertEqual(b'', self.stream.read(1)) def test_large_read(self): output = self.stream.read(1024) self.assertEqual(self.expected, output) self.assertEqual(b'', self.stream.read(1)) def test_seek_zero(self): output0 = self.stream.read() self.stream.seek(0) output1 = self.stream.read() self.assertEqual(output0, output1) b2-sdk-python-1.17.3/test/unit/v0/test_raw_api.py000066400000000000000000000154761426424117700215470ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_raw_api.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from ..test_base import TestBase from .deps import EncryptionAlgorithm from .deps import EncryptionKey from .deps import EncryptionMode from .deps import EncryptionSetting from .deps import B2RawHTTPApi from .deps import B2Http from .deps import BucketRetentionSetting, RetentionPeriod, RetentionMode from .deps_exception import UnusableFileName, WrongEncryptionModeForBucketDefault # Unicode characters for testing filenames. (0x0394 is a letter Delta.) TWO_BYTE_UNICHR = chr(0x0394) CHAR_UNDER_32 = chr(31) DEL_CHAR = chr(127) class TestRawAPIFilenames(TestBase): """Test that the filename checker passes conforming names and rejects those that don't.""" def setUp(self): self.raw_api = B2RawHTTPApi(B2Http()) def _should_be_ok(self, filename): """Call with test filenames that follow the filename rules. :param filename: unicode (or str) that follows the rules """ print(u"Filename \"{0}\" should be OK".format(filename)) self.assertTrue(self.raw_api.check_b2_filename(filename) is None) def _should_raise(self, filename, exception_message): """Call with filenames that don't follow the rules (so the rule checker should raise). :param filename: unicode (or str) that doesn't follow the rules :param exception_message: regexp that matches the exception's detailed message """ print( u"Filename \"{0}\" should raise UnusableFileName(\".*{1}.*\").".format( filename, exception_message ) ) with self.assertRaisesRegexp(UnusableFileName, exception_message): self.raw_api.check_b2_filename(filename) def test_b2_filename_checker(self): """Test a conforming and non-conforming filename for each rule. From the B2 docs (https://www.backblaze.com/b2/docs/files.html): - Names can be pretty much any UTF-8 string up to 1024 bytes long. - No character codes below 32 are allowed. - Backslashes are not allowed. - DEL characters (127) are not allowed. - File names cannot start with "/", end with "/", or contain "//". - Maximum of 250 bytes of UTF-8 in each segment (part between slashes) of a file name. """ print("test b2 filename rules") # Examples from doc: self._should_be_ok('Kitten Videos') self._should_be_ok(u'\u81ea\u7531.txt') # Check length # 1024 bytes is ok if the segments are at most 250 chars. s_1024 = 4 * (250 * 'x' + '/') + 20 * 'y' self._should_be_ok(s_1024) # 1025 is too long. self._should_raise(s_1024 + u'x', "too long") # 1024 bytes with two byte characters should also work. s_1024_two_byte = 4 * (125 * TWO_BYTE_UNICHR + u'/') + 20 * u'y' self._should_be_ok(s_1024_two_byte) # But 1025 bytes is too long. self._should_raise(s_1024_two_byte + u'x', "too long") # Names with unicode values < 32, and DEL aren't allowed. self._should_raise(u'hey' + CHAR_UNDER_32, "contains code.*less than 32") # Unicode in the filename shouldn't break the exception message. self._should_raise(TWO_BYTE_UNICHR + CHAR_UNDER_32, "contains code.*less than 32") self._should_raise(DEL_CHAR, "DEL.*not allowed") # Names can't start or end with '/' or contain '//' self._should_raise(u'/hey', "not start.*/") self._should_raise(u'hey/', "not .*end.*/") self._should_raise(u'not//allowed', "contain.*//") # Reject segments longer than 250 bytes self._should_raise(u'foo/' + 251 * u'x', "segment too long") # So a segment of 125 two-byte chars plus one should also fail. self._should_raise(u'foo/' + 125 * TWO_BYTE_UNICHR + u'x', "segment too long") class BucketTestBase: @pytest.fixture(autouse=True) def init(self, mocker): b2_http = mocker.MagicMock() self.raw_api = B2RawHTTPApi(b2_http) class TestUpdateBucket(BucketTestBase): """Test updating bucket.""" @pytest.fixture(autouse=True) def init(self, mocker): b2_http = mocker.MagicMock() self.raw_api = B2RawHTTPApi(b2_http) def test_assertion_raises(self): with pytest.raises(AssertionError): self.raw_api.update_bucket('test', 'account_auth_token', 'account_id', 'bucket_id') @pytest.mark.parametrize( 'bucket_type,bucket_info,default_retention', ( (None, {}, None), ( 'allPublic', None, BucketRetentionSetting(RetentionMode.COMPLIANCE, RetentionPeriod(years=1)) ), ) ) def test_assertion_not_raises(self, bucket_type, bucket_info, default_retention): self.raw_api.update_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', bucket_type=bucket_type, bucket_info=bucket_info, default_retention=default_retention, ) @pytest.mark.parametrize( 'encryption_setting,', ( EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(b'key', 'key-id') ), EncryptionSetting(mode=EncryptionMode.UNKNOWN,), ) ) def test_update_bucket_wrong_encryption(self, encryption_setting): with pytest.raises(WrongEncryptionModeForBucketDefault): self.raw_api.update_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', default_server_side_encryption=encryption_setting, bucket_type='allPublic', ) class TestCreateBucket(BucketTestBase): """Test creating bucket.""" @pytest.mark.parametrize( 'encryption_setting,', ( EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(b'key', 'key-id') ), EncryptionSetting(mode=EncryptionMode.UNKNOWN,), ) ) def test_create_bucket_wrong_encryption(self, encryption_setting): with pytest.raises(WrongEncryptionModeForBucketDefault): self.raw_api.create_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', bucket_type='allPrivate', bucket_info={}, default_server_side_encryption=encryption_setting, ) b2-sdk-python-1.17.3/test/unit/v0/test_scan_policies.py000066400000000000000000000043541426424117700227310ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_scan_policies.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from ..test_base import TestBase from .deps import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from .deps_exception import InvalidArgument class TestScanPolicies(TestBase): def test_default(self): self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_directory('')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_file('')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_directory('a')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_file('a')) def test_exclude_include(self): policy = ScanPoliciesManager(exclude_file_regexes=('a', 'b'), include_file_regexes=('ab',)) self.assertTrue(policy.should_exclude_file('alfa')) self.assertTrue(policy.should_exclude_file('bravo')) self.assertFalse(policy.should_exclude_file('abend')) self.assertFalse(policy.should_exclude_file('charlie')) def test_exclude_dir(self): policy = ScanPoliciesManager(exclude_dir_regexes=('alfa', 'bravo$')) self.assertTrue(policy.should_exclude_directory('alfa')) self.assertTrue(policy.should_exclude_directory('alfa2')) self.assertTrue(policy.should_exclude_directory('alfa/hello')) self.assertTrue(policy.should_exclude_directory('bravo')) self.assertFalse(policy.should_exclude_directory('bravo2')) self.assertFalse(policy.should_exclude_directory('bravo/hello')) self.assertTrue(policy.should_exclude_file('alfa/foo')) self.assertTrue(policy.should_exclude_file('alfa2/hello/foo')) self.assertTrue(policy.should_exclude_file('alfa/hello/foo.txt')) self.assertTrue(policy.should_exclude_file('bravo/foo')) self.assertFalse(policy.should_exclude_file('bravo2/hello/foo')) self.assertTrue(policy.should_exclude_file('bravo/hello/foo.txt')) def test_include_without_exclude(self): with self.assertRaises(InvalidArgument): ScanPoliciesManager(include_file_regexes=('.*',)) b2-sdk-python-1.17.3/test/unit/v0/test_session.py000066400000000000000000000063571426424117700216060ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_session.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import unittest.mock as mock from ..test_base import TestBase from .deps_exception import InvalidAuthToken, Unauthorized from .deps import ALL_CAPABILITIES from .deps import B2Session class TestB2Session(TestBase): def setUp(self): self.account_info = mock.MagicMock() self.account_info.get_account_auth_token.return_value = 'auth_token' self.api = mock.MagicMock() self.api.account_info = self.account_info self.raw_api = mock.MagicMock() self.raw_api.get_file_info_by_id.__name__ = 'get_file_info_by_id' self.raw_api.get_file_info_by_id.side_effect = ['ok'] self.session = B2Session(self.account_info, raw_api=self.raw_api) def test_works_first_time(self): self.assertEqual('ok', self.session.get_file_info_by_id(None)) def test_works_second_time(self): self.raw_api.get_file_info_by_id.side_effect = [ InvalidAuthToken('message', 'code'), 'ok', ] self.assertEqual('ok', self.session.get_file_info_by_id(None)) def test_fails_second_time(self): self.raw_api.get_file_info_by_id.side_effect = [ InvalidAuthToken('message', 'code'), InvalidAuthToken('message', 'code'), ] with self.assertRaises(InvalidAuthToken): self.session.get_file_info_by_id(None) def test_app_key_info_no_info(self): self.account_info.get_allowed.return_value = dict( bucketId=None, bucketName=None, capabilities=ALL_CAPABILITIES, namePrefix=None, ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('no_go', 'code') with self.assertRaisesRegexp( Unauthorized, r'no_go for application key with no restrictions \(code\)' ): self.session.get_file_info_by_id(None) def test_app_key_info_no_info_no_message(self): self.account_info.get_allowed.return_value = dict( bucketId=None, bucketName=None, capabilities=ALL_CAPABILITIES, namePrefix=None, ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('', 'code') with self.assertRaisesRegexp( Unauthorized, r'unauthorized for application key with no restrictions \(code\)' ): self.session.get_file_info_by_id(None) def test_app_key_info_all_info(self): self.account_info.get_allowed.return_value = dict( bucketId='123456', bucketName='my-bucket', capabilities=['readFiles'], namePrefix='prefix/', ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('no_go', 'code') with self.assertRaisesRegexp( Unauthorized, r"no_go for application key with capabilities 'readFiles', restricted to bucket 'my-bucket', restricted to files that start with 'prefix/' \(code\)" ): self.session.get_file_info_by_id(None) b2-sdk-python-1.17.3/test/unit/v0/test_sync.py000066400000000000000000001022601426424117700210650ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_sync.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import concurrent.futures as futures import os import platform import sys import threading import time import unittest from unittest.mock import MagicMock, ANY import pytest from ..test_base import TestBase from .deps_exception import UnSyncableFilename, NotADirectory, UnableToCreateDirectory, EmptyDirectory, InvalidArgument, CommandError from .deps import FileVersionInfo from .deps import B2Folder, LocalFolder from .deps import LocalSyncPath, B2SyncPath from .deps import ScanPoliciesManager, DEFAULT_SCAN_MANAGER from .deps import BoundedQueueExecutor, make_folder_sync_actions, zip_folders from .deps import parse_sync_folder from .deps import TempDir DAY = 86400000 # milliseconds TODAY = DAY * 100 # an arbitrary reference time for testing def write_file(path, contents): parent = os.path.dirname(path) if not os.path.isdir(parent): os.makedirs(parent) with open(path, 'wb') as f: f.write(contents) class TestSync(TestBase): def setUp(self): self.reporter = MagicMock() class TestFolder(TestSync): __test__ = False NAMES = [ '.dot_file', 'hello.', os.path.join('hello', 'a', '1'), os.path.join('hello', 'a', '2'), os.path.join('hello', 'b'), 'hello0', os.path.join('inner', 'a.bin'), os.path.join('inner', 'a.txt'), os.path.join('inner', 'b.bin'), os.path.join('inner', 'b.txt'), os.path.join('inner', 'more', 'a.bin'), os.path.join('inner', 'more', 'a.txt'), '\u81ea\u7531', ] MOD_TIMES = {'.dot_file': TODAY - DAY, 'hello.': TODAY - DAY} def setUp(self): super(TestFolder, self).setUp() self.root_dir = '' def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False ): raise NotImplementedError def all_files(self, policies_manager): return list(self.prepare_folder().all_files(self.reporter, policies_manager)) def assert_filtered_files(self, scan_results, expected_scan_results): self.assertEqual(expected_scan_results, list(f.relative_path for f in scan_results)) self.reporter.local_access_error.assert_not_called() def test_exclusions(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.txt', 'inner/b.txt', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*\\.bin',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_all(self): expected_list = [] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclusions_inclusions(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager( exclude_file_regexes=('.*\\.bin',), include_file_regexes=('.*a\\.bin',), ) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_matches_prefix(self): expected_list = [ '.dot_file', 'hello.', 'hello/b', 'hello0', 'inner/b.bin', 'inner/b.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*a',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory(self): expected_list = [ '.dot_file', 'hello.', 'hello0', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager( exclude_dir_regexes=('hello', 'more', 'hello0'), exclude_file_regexes=('inner',) ) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory2(self): expected_list = [ '.dot_file', 'hello.', 'hello0', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_dir_regexes=('hello$', 'inner')) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory_trailing_slash_does_not_match(self): expected_list = [ '.dot_file', 'hello.', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_dir_regexes=('hello$', 'inner/')) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclusion_with_exact_match(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('hello0',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_before_in_range(self): expected_list = [ 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_modified_before=TODAY - 100) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_before_exact(self): expected_list = [ 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_modified_before=TODAY) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_after_in_range(self): expected_list = ['.dot_file', 'hello.'] polices_manager = ScanPoliciesManager(exclude_modified_after=TODAY - 100) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_after_exact(self): expected_list = ['.dot_file', 'hello.'] polices_manager = ScanPoliciesManager(exclude_modified_after=TODAY - DAY) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) class TestLocalFolder(TestFolder): __test__ = True def setUp(self): super(TestLocalFolder, self).setUp() self.temp_dir = TempDir() self.root_dir = self.temp_dir.__enter__() def tearDown(self): self.temp_dir.__exit__(*sys.exc_info()) def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False ): assert not (broken_symlink and invalid_permissions) if platform.system() == 'Windows': pytest.skip('on Windows there are some environment issues with test directory creation') if prepare_files: for relative_path in self.NAMES: self.prepare_file(relative_path) if broken_symlink: os.symlink( os.path.join(self.root_dir, 'non_existant_file'), os.path.join(self.root_dir, 'bad_symlink') ) elif invalid_permissions: os.chmod(os.path.join(self.root_dir, self.NAMES[0]), 0) return LocalFolder(self.root_dir) def prepare_file(self, relative_path): path = os.path.join(self.root_dir, relative_path) write_file(path, b'') if relative_path in self.MOD_TIMES: mod_time = int(round(self.MOD_TIMES[relative_path] / 1000)) else: mod_time = int(round(TODAY / 1000)) os.utime(path, (mod_time, mod_time)) def test_slash_sorting(self): # '/' should sort between '.' and '0' folder = self.prepare_folder() self.assertEqual(self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter))) self.reporter.local_access_error.assert_not_called() def test_broken_symlink(self): folder = self.prepare_folder(broken_symlink=True) self.assertEqual(self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter))) self.reporter.local_access_error.assert_called_once_with( os.path.join(self.root_dir, 'bad_symlink') ) def test_invalid_permissions(self): folder = self.prepare_folder(invalid_permissions=True) # tests differ depending on the user running them. "root" will # succeed in os.access(path, os.R_OK) even if the permissions of # the file are 0 as implemented on self._prepare_folder(). # use-case: running test suite inside a docker container if not os.access(os.path.join(self.root_dir, self.NAMES[0]), os.R_OK): self.assertEqual( self.NAMES[1:], list(f.relative_path for f in folder.all_files(self.reporter)) ) self.reporter.local_permission_error.assert_called_once_with( os.path.join(self.root_dir, self.NAMES[0]) ) else: self.assertEqual( self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter)) ) def test_syncable_paths(self): syncable_paths = ( ('test.txt', 'test.txt'), ('./a/test.txt', 'a/test.txt'), ('./a/../test.txt', 'test.txt') ) folder = self.prepare_folder(prepare_files=False) for syncable_path, norm_syncable_path in syncable_paths: expected = os.path.join(self.root_dir, norm_syncable_path.replace('/', os.path.sep)) self.assertEqual(expected, folder.make_full_path(syncable_path)) def test_unsyncable_paths(self): unsyncable_paths = ('../test.txt', 'a/../../test.txt', '/a/test.txt') folder = self.prepare_folder(prepare_files=False) for unsyncable_path in unsyncable_paths: with self.assertRaises(UnSyncableFilename): folder.make_full_path(unsyncable_path) class TestB2Folder(TestFolder): __test__ = True FILE_VERSION_INFOS = { os.path.join('inner', 'a.txt'): [ ( FileVersionInfo( 'a2', 'inner/a.txt', 200, 'text/plain', 'sha1', {}, 2000, 'upload' ), '' ), ( FileVersionInfo( 'a1', 'inner/a.txt', 100, 'text/plain', 'sha1', {}, 1000, 'upload' ), '' ) ], os.path.join('inner', 'b.txt'): [ ( FileVersionInfo( 'b2', 'inner/b.txt', 200, 'text/plain', 'sha1', {}, 1999, 'upload' ), '' ), ( FileVersionInfo( 'bs', 'inner/b.txt', 150, 'text/plain', 'sha1', {}, 1500, 'start' ), '' ), ( FileVersionInfo( 'b1', 'inner/b.txt', 100, 'text/plain', 'sha1', {'src_last_modified_millis': 1001}, 6666, 'upload' ), '' ) ] } def setUp(self): super(TestB2Folder, self).setUp() self.bucket = MagicMock() self.bucket.ls.return_value = [] self.api = MagicMock() self.api.get_bucket_by_name.return_value = self.bucket def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False ): if prepare_files: for relative_path in self.NAMES: self.prepare_file(relative_path, use_file_versions_info) return B2Folder('bucket-name', self.root_dir, self.api) def prepare_file(self, relative_path, use_file_versions_info=False): if use_file_versions_info and relative_path in self.FILE_VERSION_INFOS: self.bucket.ls.return_value.extend(self.FILE_VERSION_INFOS[relative_path]) return if platform.system() == 'Windows': relative_path = relative_path.replace(os.sep, '/') if relative_path in self.MOD_TIMES: self.bucket.ls.return_value.append( ( FileVersionInfo( relative_path, relative_path, 100, 'text/plain', 'sha1', {}, self.MOD_TIMES[relative_path], 'upload' ), self.root_dir ) ) else: self.bucket.ls.return_value.append( ( FileVersionInfo( relative_path, relative_path, 100, 'text/plain', 'sha1', {}, TODAY, 'upload' ), self.root_dir ) ) def test_empty(self): folder = self.prepare_folder(prepare_files=False) self.assertEqual([], list(folder.all_files(self.reporter))) def test_multiple_versions(self): # Test two files, to cover the yield within the loop, and the yield without. folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( [ "B2Path(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", "B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])" ], [ str(f) for f in folder.all_files(self.reporter) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ] ) def test_exclude_modified_multiple_versions(self): polices_manager = ScanPoliciesManager( exclude_modified_before=1001, exclude_modified_after=1999 ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( ["B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ str(f) for f in folder.all_files(self.reporter, policies_manager=polices_manager) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ] ) def test_exclude_modified_all_versions(self): polices_manager = ScanPoliciesManager( exclude_modified_before=1500, exclude_modified_after=1500 ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( [], list(folder.all_files(self.reporter, policies_manager=polices_manager)) ) # Path names not allowed to be sync'd on Windows NOT_SYNCD_ON_WINDOWS = [ 'Z:/windows/system32/drivers/etc/hosts', 'a:/Users/.default/test', r'C:\Windows\system32\drivers\mstsc.sys', ] def test_unsyncable_filenames(self): b2_folder = B2Folder('bucket-name', '', self.api) # Test a list of unsyncable file names filenames_to_test = [ '/', # absolute root path '//', '///', '/..', '/../', '/.../', '/../.', '/../..', '/a.txt', '/folder/a.txt', './folder/a.txt', # current dir relative path 'folder/./a.txt', 'folder/folder/.', 'a//b/', # double-slashes 'a///b', 'a////b', '../test', # start with parent dir '../../test', '../../abc/../test', '../../abc/../test/', '../../abc/../.test', 'a/b/c/../d', # parent dir embedded 'a//..//b../..c/', '..a/b../..c/../d..', 'a/../', 'a/../../../../../', 'a/b/c/..', r'\\', # backslash filenames r'\z', r'..\\', r'..\..', r'\..\\', r'\\..\\..', r'\\', r'\\\\', r'\\\\server\\share\\dir\\file', r'\\server\share\dir\file', r'\\?\C\Drive\temp', r'.\\//', r'..\\..//..\\\\', r'.\\a\\..\\b', r'a\\.\\b', ] if platform.system() == "Windows": filenames_to_test.extend(self.NOT_SYNCD_ON_WINDOWS) for filename in filenames_to_test: self.bucket.ls.return_value = [ (FileVersionInfo('a1', filename, 1, 'text/plain', 'sha1', {}, 1000, 'upload'), '') ] try: list(b2_folder.all_files(self.reporter)) self.fail("should have thrown UnSyncableFilename for: '%s'" % filename) except UnSyncableFilename as e: self.assertTrue(filename in str(e)) def test_syncable_filenames(self): b2_folder = B2Folder('bucket-name', '', self.api) # Test a list of syncable file names filenames_to_test = [ '', ' ', ' / ', ' ./. ', ' ../.. ', '.. / ..', r'.. \ ..', 'file.txt', '.folder/', '..folder/', '..file', r'file/ and\ folder', 'file..', '..file..', 'folder/a.txt..', '..a/b../c../..d/e../', r'folder\test', r'folder\..f..\..f\..f', r'mix/and\match/', r'a\b\c\d', ] # filenames not permitted on Windows *should* be allowed on Linux if platform.system() != "Windows": filenames_to_test.extend(self.NOT_SYNCD_ON_WINDOWS) for filename in filenames_to_test: self.bucket.ls.return_value = [ (FileVersionInfo('a1', filename, 1, 'text/plain', 'sha1', {}, 1000, 'upload'), '') ] list(b2_folder.all_files(self.reporter)) class FakeLocalFolder(LocalFolder): def __init__(self, local_sync_paths): super().__init__('folder') self.local_sync_paths = local_sync_paths def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): for single_path in self.local_sync_paths: if single_path.relative_path.endswith('/'): if policies_manager.should_exclude_b2_directory(single_path.relative_path): continue else: if policies_manager.should_exclude_local_path(single_path): continue yield single_path def make_full_path(self, name): return '/dir/' + name class FakeB2Folder(B2Folder): def __init__(self, test_files): self.file_versions = [] for test_file in test_files: self.file_versions.extend(self.file_versions_from_file_tuples(*test_file)) super().__init__('test-bucket', 'folder', MagicMock()) def get_file_versions(self): yield from iter(self.file_versions) @classmethod def file_versions_from_file_tuples(cls, name, mod_times, size=10): """ Makes FileVersion objects. Positive modification times are uploads, and negative modification times are hides. It's a hack, but it works. """ try: mod_times = iter(mod_times) except TypeError: mod_times = [mod_times] return [ FileVersionInfo( id_='id_%s_%d' % (name[0], abs(mod_time)), file_name='folder/' + name, upload_timestamp=abs(mod_time), action='upload' if 0 < mod_time else 'hide', size=size, file_info={'in_b2': 'yes'}, content_type='text/plain', content_sha1='content_sha1', ) for mod_time in mod_times ] # yapf disable @classmethod def sync_path_from_file_tuple(cls, name, mod_times, size=10): file_versions = cls.file_versions_from_file_tuples(name, mod_times, size) return B2SyncPath(name, file_versions[0], file_versions) class TestParseSyncFolder(TestBase): def test_b2_double_slash(self): self._check_one('B2Folder(my-bucket, folder/path)', 'b2://my-bucket/folder/path') def test_b2_no_double_slash(self): self._check_one('B2Folder(my-bucket, folder/path)', 'b2:my-bucket/folder/path') def test_b2_trailing_slash(self): self._check_one('B2Folder(my-bucket, a)', 'b2://my-bucket/a/') def test_b2_no_folder(self): self._check_one('B2Folder(my-bucket, )', 'b2://my-bucket') self._check_one('B2Folder(my-bucket, )', 'b2://my-bucket/') def test_local(self): if platform.system() == 'Windows': drive, _ = os.path.splitdrive(os.getcwd()) expected = 'LocalFolder(\\\\?\\%s\\foo)' % (drive,) else: expected = 'LocalFolder(/foo)' self._check_one(expected, '/foo') def test_local_trailing_slash(self): if platform.system() == 'Windows': drive, _ = os.path.splitdrive(os.getcwd()) expected = 'LocalFolder(\\\\?\\%s\\foo)' % (drive,) else: expected = 'LocalFolder(/foo)' self._check_one(expected, '/foo/') def _check_one(self, expected, to_parse): api = MagicMock() self.assertEqual(expected, str(parse_sync_folder(str(to_parse), api))) class TestFolderExceptions: """There is an exact copy of this class in unit/v1/test_sync.py - TODO: leave only one when migrating tests to sync-like structure. """ @pytest.mark.parametrize( 'exception,msg', [ pytest.param(NotADirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param(Exception, 'is not a directory', marks=pytest.mark.apiver(to_ver=1)), ], ) def test_ensure_present_not_a_dir(self, exception, msg): with TempDir() as path: file = os.path.join(path, 'clearly_a_file') with open(file, 'w') as f: f.write(' ') folder = parse_sync_folder(file, MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_present() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(UnableToCreateDirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( Exception, 'unable to create directory', marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_ensure_present_unable_to_create(self, exception, msg): with TempDir() as path: file = os.path.join(path, 'clearly_a_file') with open(file, 'w') as f: f.write(' ') folder = parse_sync_folder(os.path.join(file, 'nonsense'), MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_present() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(EmptyDirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( CommandError, 'Directory .* is empty. Use --allowEmptySource to sync anyway.', marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_ensure_non_empty(self, exception, msg): with TempDir() as path: folder = parse_sync_folder(path, MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_non_empty() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(InvalidArgument, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( CommandError, "'//' not allowed in path names", marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_double_slash_not_allowed(self, exception, msg): with pytest.raises(exception, match=msg): parse_sync_folder('b2://a//b', MagicMock()) class TestZipFolders(TestSync): def test_empty(self): folder_a = FakeB2Folder([]) folder_b = FakeB2Folder([]) self.assertEqual([], list(zip_folders(folder_a, folder_b, self.reporter))) def test_one_empty(self): file_a1 = LocalSyncPath("a.txt", "a.txt", 100, 10) folder_a = FakeLocalFolder([file_a1]) folder_b = FakeB2Folder([]) self.assertEqual([(file_a1, None)], list(zip_folders(folder_a, folder_b, self.reporter))) def test_two(self): file_a1 = ("a.txt", 100, 10) file_a2 = ("b.txt", 100, 10) file_a3 = ("d.txt", 100, 10) file_a4 = ("f.txt", 100, 10) file_b1 = ("b.txt", 200, 10) file_b2 = ("e.txt", 200, 10) folder_a = FakeB2Folder([file_a1, file_a2, file_a3, file_a4]) folder_b = FakeB2Folder([file_b1, file_b2]) self.assertEqual( [ (FakeB2Folder.sync_path_from_file_tuple(*file_a1), None), ( FakeB2Folder.sync_path_from_file_tuple(*file_a2), FakeB2Folder.sync_path_from_file_tuple(*file_b1) ), (FakeB2Folder.sync_path_from_file_tuple(*file_a3), None), (None, FakeB2Folder.sync_path_from_file_tuple(*file_b2)), (FakeB2Folder.sync_path_from_file_tuple(*file_a4), None) ], list(zip_folders(folder_a, folder_b, self.reporter)) ) def test_pass_reporter_to_folder(self): """ Check that the zip_folders() function passes the reporter through to both folders. """ folder_a = MagicMock() folder_b = MagicMock() folder_a.all_files = MagicMock(return_value=iter([])) folder_b.all_files = MagicMock(return_value=iter([])) self.assertEqual([], list(zip_folders(folder_a, folder_b, self.reporter))) folder_a.all_files.assert_called_once_with(self.reporter, ANY) folder_b.all_files.assert_called_once_with(self.reporter) class FakeArgs: """ Can be passed to sync code to simulate command-line options. """ def __init__( self, delete=False, keepDays=None, skipNewer=False, replaceNewer=False, compareVersions=None, compareThreshold=None, excludeRegex=None, excludeDirRegex=None, includeRegex=None, debugLogs=True, dryRun=False, allowEmptySource=False, excludeAllSymlinks=False, ): self.delete = delete self.keepDays = keepDays self.skipNewer = skipNewer self.replaceNewer = replaceNewer self.compareVersions = compareVersions self.compareThreshold = compareThreshold if excludeRegex is None: excludeRegex = [] self.excludeRegex = excludeRegex if includeRegex is None: includeRegex = [] self.includeRegex = includeRegex if excludeDirRegex is None: excludeDirRegex = [] self.excludeDirRegex = excludeDirRegex self.debugLogs = debugLogs self.dryRun = dryRun self.allowEmptySource = allowEmptySource self.excludeAllSymlinks = excludeAllSymlinks def local_file(name, mod_time, size=10): """ Makes a File object for a b2 file, with one FileVersion for each modification time given in mod_times. """ return LocalSyncPath(name, name, mod_time, size) class TestExclusions(TestSync): def _check_folder_sync(self, expected_actions, fakeargs): # only local file_a = ('a.txt', 100) file_b = ('b.txt', 100) file_d = ('d/d.txt', 100) file_e = ('e/e.incl', 100) # both local and remote file_bi = ('b.txt.incl', 100) file_z = ('z.incl', 100) # only remote file_c = ('c.txt', 100) local_folder = FakeLocalFolder( [local_file(*f) for f in (file_a, file_b, file_d, file_e, file_bi, file_z)] ) b2_folder = FakeB2Folder([file_bi, file_c, file_z]) policies_manager = ScanPoliciesManager( exclude_dir_regexes=fakeargs.excludeDirRegex, exclude_file_regexes=fakeargs.excludeRegex, include_file_regexes=fakeargs.includeRegex, exclude_all_symlinks=fakeargs.excludeAllSymlinks ) actions = list( make_folder_sync_actions( local_folder, b2_folder, fakeargs, TODAY, self.reporter, policies_manager ) ) self.assertEqual(expected_actions, [str(a) for a in actions]) def test_file_exclusions_with_delete(self): expected_actions = [ 'b2_upload(/dir/a.txt, folder/a.txt, 100)', 'b2_delete(folder/b.txt.incl, id_b_100, )', 'b2_delete(folder/c.txt, id_c_100, )', 'b2_upload(/dir/d/d.txt, folder/d/d.txt, 100)', 'b2_upload(/dir/e/e.incl, folder/e/e.incl, 100)', ] self._check_folder_sync(expected_actions, FakeArgs(delete=True, excludeRegex=["b\\.txt"])) def test_file_exclusions_inclusions_with_delete(self): expected_actions = [ 'b2_upload(/dir/a.txt, folder/a.txt, 100)', 'b2_delete(folder/b.txt.incl, id_b_100, )', 'b2_delete(folder/c.txt, id_c_100, )', 'b2_upload(/dir/d/d.txt, folder/d/d.txt, 100)', 'b2_upload(/dir/e/e.incl, folder/e/e.incl, 100)', 'b2_upload(/dir/b.txt.incl, folder/b.txt.incl, 100)', ] fakeargs = FakeArgs(delete=True, excludeRegex=["b\\.txt"], includeRegex=[".*\\.incl"]) self._check_folder_sync(expected_actions, fakeargs) class TestBoundedQueueExecutor(TestBase): def test_run_more_than_queue_size(self): """ Makes sure that the executor will run more jobs that the queue size, which ensures that the semaphore gets released, even if an exception is thrown. """ raw_executor = futures.ThreadPoolExecutor(1) bounded_executor = BoundedQueueExecutor(raw_executor, 5) class Counter: """ Counts how many times run() is called. """ def __init__(self): self.counter = 0 def run(self): """ Always increments the counter. Sometimes raises an exception. """ self.counter += 1 if self.counter % 2 == 0: raise Exception('test') counter = Counter() for _ in range(10): bounded_executor.submit(counter.run) bounded_executor.shutdown() self.assertEqual(10, counter.counter) def test_wait_for_running_jobs(self): """ Makes sure that no more than queue_limit workers are running at once, which checks that the semaphore is acquired before submitting an action. """ raw_executor = futures.ThreadPoolExecutor(2) bounded_executor = BoundedQueueExecutor(raw_executor, 1) assert_equal = self.assertEqual class CountAtOnce: """ Counts how many threads are running at once. There should never be more than 1 because that's the limit on the bounded executor. """ def __init__(self): self.running_at_once = 0 self.lock = threading.Lock() def run(self): with self.lock: self.running_at_once += 1 assert_equal(1, self.running_at_once) # While we are sleeping here, no other actions should start # running. If they do, they will increment the counter and # fail the above assertion. time.sleep(0.05) with self.lock: self.running_at_once -= 1 self.counter += 1 if self.counter % 2 == 0: raise Exception('test') count_at_once = CountAtOnce() for _ in range(5): bounded_executor.submit(count_at_once.run) bounded_executor.shutdown() if __name__ == '__main__': unittest.main() b2-sdk-python-1.17.3/test/unit/v0/test_utils.py000066400000000000000000000154721426424117700212610ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_utils.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from ..test_base import TestBase from .deps import b2_url_encode, b2_url_decode, choose_part_ranges, format_and_scale_number, format_and_scale_fraction # These are from the B2 Docs (https://www.backblaze.com/b2/docs/string_encoding.html) ENCODING_TEST_CASES = [ { 'fullyEncoded': '%20', 'minimallyEncoded': '+', 'string': ' ' }, { 'fullyEncoded': '%21', 'minimallyEncoded': '!', 'string': '!' }, { 'fullyEncoded': '%22', 'minimallyEncoded': '%22', 'string': '"' }, { 'fullyEncoded': '%23', 'minimallyEncoded': '%23', 'string': '#' }, { 'fullyEncoded': '%24', 'minimallyEncoded': '$', 'string': '$' }, { 'fullyEncoded': '%25', 'minimallyEncoded': '%25', 'string': '%' }, { 'fullyEncoded': '%26', 'minimallyEncoded': '%26', 'string': '&' }, { 'fullyEncoded': '%27', 'minimallyEncoded': "'", 'string': "'" }, { 'fullyEncoded': '%28', 'minimallyEncoded': '(', 'string': '(' }, { 'fullyEncoded': '%29', 'minimallyEncoded': ')', 'string': ')' }, { 'fullyEncoded': '%2A', 'minimallyEncoded': '*', 'string': '*' }, { 'fullyEncoded': '%2B', 'minimallyEncoded': '%2B', 'string': '+' }, { 'fullyEncoded': '%2C', 'minimallyEncoded': '%2C', 'string': ',' }, { 'fullyEncoded': '%2D', 'minimallyEncoded': '-', 'string': '-' }, { 'fullyEncoded': '%2E', 'minimallyEncoded': '.', 'string': '.' }, { 'fullyEncoded': '/', 'minimallyEncoded': '/', 'string': '/' }, { 'fullyEncoded': '%30', 'minimallyEncoded': '0', 'string': '0' }, { 'fullyEncoded': '%39', 'minimallyEncoded': '9', 'string': '9' }, { 'fullyEncoded': '%3A', 'minimallyEncoded': ':', 'string': ':' }, { 'fullyEncoded': '%3B', 'minimallyEncoded': ';', 'string': ';' }, { 'fullyEncoded': '%3C', 'minimallyEncoded': '%3C', 'string': '<' }, { 'fullyEncoded': '%3D', 'minimallyEncoded': '=', 'string': '=' }, { 'fullyEncoded': '%3E', 'minimallyEncoded': '%3E', 'string': '>' }, { 'fullyEncoded': '%3F', 'minimallyEncoded': '%3F', 'string': '?' }, { 'fullyEncoded': '%40', 'minimallyEncoded': '@', 'string': '@' }, { 'fullyEncoded': '%41', 'minimallyEncoded': 'A', 'string': 'A' }, { 'fullyEncoded': '%5A', 'minimallyEncoded': 'Z', 'string': 'Z' }, { 'fullyEncoded': '%5B', 'minimallyEncoded': '%5B', 'string': '[' }, { 'fullyEncoded': '%5C', 'minimallyEncoded': '%5C', 'string': '\\' }, { 'fullyEncoded': '%5D', 'minimallyEncoded': '%5D', 'string': ']' }, { 'fullyEncoded': '%5E', 'minimallyEncoded': '%5E', 'string': '^' }, { 'fullyEncoded': '%5F', 'minimallyEncoded': '_', 'string': '_' }, { 'fullyEncoded': '%60', 'minimallyEncoded': '%60', 'string': '`' }, { 'fullyEncoded': '%61', 'minimallyEncoded': 'a', 'string': 'a' }, { 'fullyEncoded': '%7A', 'minimallyEncoded': 'z', 'string': 'z' }, { 'fullyEncoded': '%7B', 'minimallyEncoded': '%7B', 'string': '{' }, { 'fullyEncoded': '%7C', 'minimallyEncoded': '%7C', 'string': '|' }, { 'fullyEncoded': '%7D', 'minimallyEncoded': '%7D', 'string': '}' }, { 'fullyEncoded': '%7E', 'minimallyEncoded': '~', 'string': '~' }, { 'fullyEncoded': '%7F', 'minimallyEncoded': '%7F', 'string': u'\u007f' }, { 'fullyEncoded': '%E8%87%AA%E7%94%B1', 'minimallyEncoded': '%E8%87%AA%E7%94%B1', 'string': u'\u81ea\u7531' }, { 'fullyEncoded': '%F0%90%90%80', 'minimallyEncoded': '%F0%90%90%80', 'string': u'\U00010400' }, ] class TestUrlEncoding(TestBase): def test_it(self): for test_case in ENCODING_TEST_CASES: string = test_case['string'] fully_encoded = test_case['fullyEncoded'] minimally_encoded = test_case['minimallyEncoded'] encoded = b2_url_encode(string) expected_encoded = (minimally_encoded, fully_encoded) if encoded not in expected_encoded: print( 'string: %s encoded: %s expected: %s' % (repr(string), encoded, expected_encoded) ) self.assertTrue(encoded in expected_encoded) self.assertEqual(string, b2_url_decode(fully_encoded)) self.assertEqual(string, b2_url_decode(minimally_encoded)) class TestChooseParts(TestBase): def test_it(self): self._check_one([(0, 100), (100, 100)], 200, 100) self._check_one([(0, 149), (149, 150)], 299, 100) self._check_one([(0, 100), (100, 100), (200, 100)], 300, 100) ten_TB = 10 * 1000 * 1000 * 1000 * 1000 one_GB = 1000 * 1000 * 1000 expected = [(i * one_GB, one_GB) for i in range(10000)] actual = choose_part_ranges(ten_TB, 100 * 1000 * 1000) self.assertEqual(expected, actual) def _check_one(self, expected, content_length, min_part_size): self.assertEqual(expected, choose_part_ranges(content_length, min_part_size)) class TestFormatAndScaleNumber(TestBase): def test_it(self): self._check_one('1 B', 1) self._check_one('999 B', 999) self._check_one('1.00 kB', 1000) self._check_one('999 kB', 999000) def _check_one(self, expected, x): self.assertEqual(expected, format_and_scale_number(x, 'B')) class TestFormatAndScaleFraction(TestBase): def test_it(self): self._check_one('0 / 100 B', 0, 100) self._check_one('0.0 / 10.0 kB', 0, 10000) self._check_one('9.4 / 10.0 kB', 9400, 10000) def _check_one(self, expected, numerator, denominator): self.assertEqual(expected, format_and_scale_fraction(numerator, denominator, 'B')) b2-sdk-python-1.17.3/test/unit/v0/test_version_utils.py000066400000000000000000000077401426424117700230250ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_version_utils.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import warnings from ..test_base import TestBase from .deps import rename_argument, rename_function class TestRenameArgument(TestBase): VERSION = '0.1.10' def test_warning(self): @rename_argument('aaa', 'bbb', '0.1.0', '0.2.0', current_version=self.VERSION) def easy(bbb): """ easy docstring """ return bbb # check that warning is not emitted too early with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") assert easy(5) == 5 assert easy(bbb=5) == 5 assert easy.__name__ == 'easy' assert easy.__doc__ == ' easy docstring ' assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") assert easy(aaa=5) == 5 assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert str( w[-1].message ) == "'aaa' is a deprecated argument for 'easy' function/method - it was renamed to 'bbb' in version 0.1.0. Support for the old name is going to be dropped in 0.2.0.", str( w[-1].message ) def test_outdated_replacement(self): with self.assertRaises( AssertionError, msg= "rename_argument decorator is still used in version %s when old argument name 'aaa' was scheduled to be dropped in 0.1.2. It is time to remove the mapping." % (self.VERSION,), ): @rename_argument('aaa', 'bbb', '0.1.0', '0.1.2', current_version=self.VERSION) def late(bbb): return bbb assert late # make linters happy def test_future_replacement(self): with self.assertRaises( AssertionError, msg= "rename_argument decorator indicates that the replacement of argument 'aaa' should take place in the future version 0.2.0, while the current version is 0.2.2. It looks like should be _discouraged_ at this point and not _deprecated_ yet. Consider using 'discourage_argument' decorator instead." ): @rename_argument('aaa', 'bbb', '0.2.0', '0.2.2', current_version=self.VERSION) def early(bbb): return bbb assert early # make linters happy def test_inverted_versions(self): with self.assertRaises( AssertionError, msg= "rename_argument decorator is set to start renaming argument 'aaa' starting at version 0.2.2 and finishing in 0.2.0. It needs to start at a lower version and finish at a higher version." ): @rename_argument('aaa', 'bbb', '0.2.2', '0.2.0', current_version=self.VERSION) def backwards(bbb): return bbb assert backwards # make linters happy class TestRenameFunction(TestBase): VERSION = '0.1.10' def test_rename_function(self): def new(bbb): return bbb for i in ('new', new): @rename_function(i, '0.1.0', '0.2.0', current_version=self.VERSION) def old(bbb): return bbb with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") assert old(5) == 5 assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert str( w[-1].message ) == "'old' is deprecated since version 0.1.0 - it was moved to 'new', please switch to use that. The proxy for the old name is going to be removed in 0.2.0.", str( w[-1].message ) b2-sdk-python-1.17.3/test/unit/v1/000077500000000000000000000000001426424117700165005ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/v1/__init__.py000066400000000000000000000004431426424117700206120ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/v1/apiver/000077500000000000000000000000001426424117700177665ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/v1/apiver/__init__.py000066400000000000000000000006221426424117700220770ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/apiver/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details b2-sdk-python-1.17.3/test/unit/v1/apiver/apiver_deps.py000066400000000000000000000005241426424117700226420ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/apiver/apiver_deps.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.v1 import * # noqa V = 1 b2-sdk-python-1.17.3/test/unit/v1/apiver/apiver_deps_exception.py000066400000000000000000000005411426424117700247170ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/apiver/apiver_deps_exception.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.v1.exception import * # noqa b2-sdk-python-1.17.3/test/unit/v1/deps.py000066400000000000000000000011311426424117700200010ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/deps.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # TODO: This module is used in old-style unit tests, written separately for v0 and v1. # It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details from apiver_deps import * b2-sdk-python-1.17.3/test/unit/v1/deps_exception.py000066400000000000000000000011551426424117700220650ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/deps_exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # TODO: This module is used in old-style unit tests, written separately for v0 and v1. # It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details from apiver_deps_exception import * b2-sdk-python-1.17.3/test/unit/v1/test_bounded_queue_executor.py000066400000000000000000000037551426424117700246650ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_bounded_queue_executor.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import concurrent.futures as futures import time from .deps import BoundedQueueExecutor from ..test_base import TestBase class TestBoundedQueueExecutor(TestBase): def setUp(self): unbounded_executor = futures.ThreadPoolExecutor(max_workers=1) self.executor = BoundedQueueExecutor(unbounded_executor, 1) def tearDown(self): self.executor.shutdown() def test_return_future(self): future_1 = self.executor.submit(lambda: 1) print(future_1) self.assertEqual(1, future_1.result()) def test_blocking(self): # This doesn't actually test that it waits, but it does exercise the code. # Make some futures using a function that takes a little time. def sleep_and_return_fcn(n): def fcn(): time.sleep(0.01) return n return fcn futures = [self.executor.submit(sleep_and_return_fcn(i)) for i in range(10)] # Check the answers answers = list(map(lambda f: f.result(), futures)) self.assertEqual(list(range(10)), answers) def test_no_exceptions(self): f = self.executor.submit(lambda: 1) self.executor.shutdown() self.assertEqual(0, self.executor.get_num_exceptions()) self.assertTrue(f.exception() is None) def test_two_exceptions(self): def thrower(): raise Exception('test_exception') f1 = self.executor.submit(thrower) f2 = self.executor.submit(thrower) self.executor.shutdown() self.assertEqual(2, self.executor.get_num_exceptions()) self.assertFalse(f1.exception() is None) self.assertEqual('test_exception', str(f2.exception())) b2-sdk-python-1.17.3/test/unit/v1/test_copy_manager.py000066400000000000000000000140601426424117700225560ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_copy_manager.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from ..test_base import TestBase from .deps_exception import InvalidAuthToken, Unauthorized, SSECKeyIdMismatchInCopy from .deps import MetadataDirectiveMode from .deps import EncryptionAlgorithm, EncryptionSetting, EncryptionMode, EncryptionKey, SSE_NONE, SSE_B2_AES from b2sdk.transfer.outbound.copy_manager import CopyManager from b2sdk.http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_key', key_id='some-id'), ) SSE_C_AES_2 = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_other_key', key_id='some-id-2'), ) class TestCopyManager(TestBase): def test_establish_sse_c_replace(self): file_info = {'some_key': 'some_value'} content_type = 'text/plain' metadata_directive, new_file_info, new_content_type = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.REPLACE, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info=None, source_content_type=None, ) self.assertEqual( ( MetadataDirectiveMode.REPLACE, { 'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id' }, content_type ), (metadata_directive, new_file_info, new_content_type) ) def test_establish_sse_c_copy_no_enc(self): file_info = {} content_type = 'text/plain' metadata_directive, new_file_info, new_content_type = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=None, source_server_side_encryption=None, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, {}, content_type), (metadata_directive, new_file_info, new_content_type) ) def test_establish_sse_c_copy_b2(self): file_info = {} content_type = 'text/plain' metadata_directive, new_file_info, new_content_type = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_B2_AES, source_server_side_encryption=None, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, {}, content_type), (metadata_directive, new_file_info, new_content_type) ) def test_establish_sse_c_copy_same_key_id(self): file_info = None content_type = 'text/plain' metadata_directive, new_file_info, new_content_type = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, None, content_type), (metadata_directive, new_file_info, new_content_type) ) def test_establish_sse_c_copy_sources_given(self): metadata_directive, new_file_info, new_content_type = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=None, destination_content_type=None, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info={ 'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id-2' }, source_content_type='text/plain', ) self.assertEqual( ( MetadataDirectiveMode.REPLACE, { 'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id' }, 'text/plain' ), (metadata_directive, new_file_info, new_content_type) ) def test_establish_sse_c_copy_sources_unknown(self): for source_file_info, source_content_type in [ (None, None), ({ 'a': 'b' }, None), (None, 'text/plain'), ]: with self.subTest( source_file_info=source_file_info, source_content_type=source_content_type ): with self.assertRaises( SSECKeyIdMismatchInCopy, 'attempting to copy file using MetadataDirectiveMode.COPY without providing source_file_info ' 'and source_content_type for differing sse_c_key_ids: source="some-id-2", destination="some-id"' ): CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=None, destination_content_type=None, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info=source_file_info, source_content_type=source_content_type, ) b2-sdk-python-1.17.3/test/unit/v1/test_download_dest.py000066400000000000000000000070711426424117700227440ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_download_dest.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os from ..test_base import TestBase from .deps import DownloadDestLocalFile, DownloadDestProgressWrapper, PreSeekedDownloadDest from .deps import ProgressListenerForTest from .deps import TempDir class TestDownloadDestLocalFile(TestBase): expected_result = 'hello world' def _make_dest(self, temp_dir): file_path = os.path.join(temp_dir, "test.txt") return DownloadDestLocalFile(file_path), file_path def test_write_and_set_mod_time(self): """ Check that the file gets written and that its mod time gets set. """ mod_time = 1500222333000 with TempDir() as temp_dir: download_dest, file_path = self._make_dest(temp_dir) with download_dest.make_file_context( "file_id", "file_name", 100, "content_type", "sha1", {}, mod_time ) as f: f.write(b'hello world') with open(file_path, 'rb') as f: self.assertEqual( self.expected_result.encode(), f.read(), ) self.assertEqual(mod_time, int(os.path.getmtime(file_path) * 1000)) def test_failed_write_deletes_partial_file(self): with TempDir() as temp_dir: download_dest, file_path = self._make_dest(temp_dir) try: with download_dest.make_file_context( "file_id", "file_name", 100, "content_type", "sha1", {}, 1500222333000 ) as f: f.write(b'hello world') raise Exception('test error') except Exception as e: self.assertEqual('test error', str(e)) self.assertFalse(os.path.exists(file_path), msg='failed download should be deleted') class TestPreSeekedDownloadDest(TestDownloadDestLocalFile): expected_result = '123hello world567890' def _make_dest(self, temp_dir): file_path = os.path.join(temp_dir, "test.txt") with open(file_path, 'wb') as f: f.write(b'12345678901234567890') return PreSeekedDownloadDest(local_file_path=file_path, seek_target=3), file_path class TestDownloadDestProgressWrapper(TestBase): def test_write_and_set_mod_time_and_progress(self): """ Check that the file gets written and that its mod time gets set. """ mod_time = 1500222333000 with TempDir() as temp_dir: file_path = os.path.join(temp_dir, "test.txt") download_local_file = DownloadDestLocalFile(file_path) progress_listener = ProgressListenerForTest() download_dest = DownloadDestProgressWrapper(download_local_file, progress_listener) with download_dest.make_file_context( "file_id", "file_name", 100, "content_type", "sha1", {}, mod_time ) as f: f.write(b'hello world\n') with open(file_path, 'rb') as f: self.assertEqual(b'hello world\n', f.read()) self.assertEqual(mod_time, int(os.path.getmtime(file_path) * 1000)) self.assertEqual( [ 'set_total_bytes(100)', 'bytes_completed(12)', 'close()', ], progress_listener.get_calls(), ) b2-sdk-python-1.17.3/test/unit/v1/test_file_metadata.py000066400000000000000000000030301426424117700226640ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_file_metadata.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from .deps import FileMetadata from ..test_base import TestBase def snake_to_camel(name): camel = ''.join(s.title() for s in name.split('_')) return camel[:1].lower() + camel[1:] class TestFileMetadata(TestBase): KWARGS = { 'file_id': '4_deadbeaf3b3e38a957f100d1e_f1042665d79618ae7_d20200903_m194254_c000_v0001053_t0048', 'file_name': 'foo.txt', 'content_type': 'text/plain', 'content_length': '1', 'content_sha1': '4518012e1b365e504001dbc94120624f15b8bbd5', 'file_info': {}, } # yapf: disable INFO_DICT = {snake_to_camel(k): v for k, v in KWARGS.items()} def test_verified_sha1(self): metadata = FileMetadata(**self.KWARGS) self.assertTrue(metadata.content_sha1_verified) self.assertEqual(metadata.as_info_dict(), self.INFO_DICT) def test_unverified_sha1(self): kwargs = self.KWARGS.copy() kwargs['content_sha1'] = 'unverified:' + kwargs['content_sha1'] info_dict = self.INFO_DICT.copy() info_dict['contentSha1'] = 'unverified:' + info_dict['contentSha1'] metadata = FileMetadata(**kwargs) self.assertFalse(metadata.content_sha1_verified) self.assertEqual(metadata.as_info_dict(), info_dict) b2-sdk-python-1.17.3/test/unit/v1/test_policy.py000066400000000000000000000063621426424117700214170ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_policy.py # # Copyright 2019, Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from unittest.mock import MagicMock from ..test_base import TestBase from .deps import FileVersionInfo from .deps import LocalSyncPath, B2SyncPath from .deps import B2Folder from .deps import make_b2_keep_days_actions class TestMakeB2KeepDaysActions(TestBase): def setUp(self): self.keep_days = 7 self.today = 100 * 86400 self.one_day_millis = 86400 * 1000 def test_no_versions(self): self.check_one_answer(True, [], []) def test_new_version_no_action(self): self.check_one_answer(True, [(1, -5, 'upload')], []) def test_no_source_one_old_version_hides(self): # An upload that is old gets deleted if there is no source file. self.check_one_answer(False, [(1, -10, 'upload')], ['b2_hide(folder/a)']) def test_old_hide_causes_delete(self): # A hide marker that is old gets deleted, as do the things after it. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'hide'), (3, -20, 'upload')], ['b2_delete(folder/a, 2, (hide marker))', 'b2_delete(folder/a, 3, (old version))'] ) def test_old_upload_causes_delete(self): # An upload that is old stays if there is a source file, but things # behind it go away. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'upload'), (3, -20, 'upload')], ['b2_delete(folder/a, 3, (old version))'] ) def test_out_of_order_dates(self): # The one at date -3 will get deleted because the one before it is old. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'upload'), (3, -3, 'upload')], ['b2_delete(folder/a, 3, (old version))'] ) def check_one_answer(self, has_source, id_relative_date_action_list, expected_actions): source_file = LocalSyncPath('a', 'a', 100, 10) if has_source else None dest_file_versions = [ FileVersionInfo( id_=id_, file_name='folder/' + 'a', upload_timestamp=self.today + relative_date * self.one_day_millis, action=action, size=100, file_info={}, content_type='text/plain', content_sha1='content_sha1', ) for (id_, relative_date, action) in id_relative_date_action_list ] dest_file = B2SyncPath( 'a', selected_version=dest_file_versions[0], all_versions=dest_file_versions ) if dest_file_versions else None bucket = MagicMock() api = MagicMock() api.get_bucket_by_name.return_value = bucket dest_folder = B2Folder('bucket-1', 'folder', api) actual_actions = list( make_b2_keep_days_actions( source_file, dest_file, dest_folder, dest_folder, self.keep_days, self.today ) ) actual_action_strs = [str(a) for a in actual_actions] self.assertEqual(expected_actions, actual_action_strs) b2-sdk-python-1.17.3/test/unit/v1/test_progress.py000066400000000000000000000037671426424117700217720ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_progress.py # # Copyright 2019, Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from io import BytesIO from ..test_base import TestBase from .deps import StreamWithHash from .deps import hex_sha1_of_bytes class TestHashingStream(TestBase): def setUp(self): self.data = b'01234567' self.stream = StreamWithHash(BytesIO(self.data)) self.hash = hex_sha1_of_bytes(self.data) self.expected = self.data + self.hash.encode() def test_no_argument(self): output = self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_less(self): output = self.stream.read(len(self.data) - 1) self.assertEqual(len(output), len(self.data) - 1) output += self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_equal(self): output = self.stream.read(len(self.data)) self.assertEqual(len(output), len(self.data)) output += self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_more(self): output = self.stream.read(len(self.data) + 1) self.assertEqual(len(output), len(self.data) + 1) output += self.stream.read() self.assertEqual(self.expected, output) def test_one_by_one(self): for expected_byte in self.expected: self.assertEqual(bytes((expected_byte,)), self.stream.read(1)) self.assertEqual(b'', self.stream.read(1)) def test_large_read(self): output = self.stream.read(1024) self.assertEqual(self.expected, output) self.assertEqual(b'', self.stream.read(1)) def test_seek_zero(self): output0 = self.stream.read() self.stream.seek(0) output1 = self.stream.read() self.assertEqual(output0, output1) b2-sdk-python-1.17.3/test/unit/v1/test_raw_api.py000066400000000000000000000152551426424117700215430ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_raw_api.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from ..test_base import TestBase from .deps import EncryptionAlgorithm from .deps import EncryptionKey from .deps import EncryptionMode from .deps import EncryptionSetting from .deps import B2RawHTTPApi from .deps import B2Http from .deps import BucketRetentionSetting, RetentionPeriod, RetentionMode from .deps_exception import UnusableFileName, WrongEncryptionModeForBucketDefault # Unicode characters for testing filenames. (0x0394 is a letter Delta.) TWO_BYTE_UNICHR = chr(0x0394) CHAR_UNDER_32 = chr(31) DEL_CHAR = chr(127) class TestRawAPIFilenames(TestBase): """Test that the filename checker passes conforming names and rejects those that don't.""" def setUp(self): self.raw_api = B2RawHTTPApi(B2Http()) def _should_be_ok(self, filename): """Call with test filenames that follow the filename rules. :param filename: unicode (or str) that follows the rules """ print(u"Filename \"{0}\" should be OK".format(filename)) self.assertTrue(self.raw_api.check_b2_filename(filename) is None) def _should_raise(self, filename, exception_message): """Call with filenames that don't follow the rules (so the rule checker should raise). :param filename: unicode (or str) that doesn't follow the rules :param exception_message: regexp that matches the exception's detailed message """ print( u"Filename \"{0}\" should raise UnusableFileName(\".*{1}.*\").".format( filename, exception_message ) ) with self.assertRaisesRegexp(UnusableFileName, exception_message): self.raw_api.check_b2_filename(filename) def test_b2_filename_checker(self): """Test a conforming and non-conforming filename for each rule. From the B2 docs (https://www.backblaze.com/b2/docs/files.html): - Names can be pretty much any UTF-8 string up to 1024 bytes long. - No character codes below 32 are allowed. - Backslashes are not allowed. - DEL characters (127) are not allowed. - File names cannot start with "/", end with "/", or contain "//". - Maximum of 250 bytes of UTF-8 in each segment (part between slashes) of a file name. """ print("test b2 filename rules") # Examples from doc: self._should_be_ok('Kitten Videos') self._should_be_ok(u'\u81ea\u7531.txt') # Check length # 1024 bytes is ok if the segments are at most 250 chars. s_1024 = 4 * (250 * 'x' + '/') + 20 * 'y' self._should_be_ok(s_1024) # 1025 is too long. self._should_raise(s_1024 + u'x', "too long") # 1024 bytes with two byte characters should also work. s_1024_two_byte = 4 * (125 * TWO_BYTE_UNICHR + u'/') + 20 * u'y' self._should_be_ok(s_1024_two_byte) # But 1025 bytes is too long. self._should_raise(s_1024_two_byte + u'x', "too long") # Names with unicode values < 32, and DEL aren't allowed. self._should_raise(u'hey' + CHAR_UNDER_32, "contains code.*less than 32") # Unicode in the filename shouldn't break the exception message. self._should_raise(TWO_BYTE_UNICHR + CHAR_UNDER_32, "contains code.*less than 32") self._should_raise(DEL_CHAR, "DEL.*not allowed") # Names can't start or end with '/' or contain '//' self._should_raise(u'/hey', "not start.*/") self._should_raise(u'hey/', "not .*end.*/") self._should_raise(u'not//allowed', "contain.*//") # Reject segments longer than 250 bytes self._should_raise(u'foo/' + 251 * u'x', "segment too long") # So a segment of 125 two-byte chars plus one should also fail. self._should_raise(u'foo/' + 125 * TWO_BYTE_UNICHR + u'x', "segment too long") class BucketTestBase: @pytest.fixture(autouse=True) def init(self, mocker): b2_http = mocker.MagicMock() self.raw_api = B2RawHTTPApi(b2_http) class TestUpdateBucket(BucketTestBase): """Test updating bucket.""" def test_assertion_raises(self): with pytest.raises(AssertionError): self.raw_api.update_bucket('test', 'account_auth_token', 'account_id', 'bucket_id') @pytest.mark.parametrize( 'bucket_type,bucket_info,default_retention', ( (None, {}, None), ( 'allPublic', None, BucketRetentionSetting(RetentionMode.COMPLIANCE, RetentionPeriod(years=1)) ), ) ) def test_assertion_not_raises(self, bucket_type, bucket_info, default_retention): self.raw_api.update_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', bucket_type=bucket_type, bucket_info=bucket_info, default_retention=default_retention, ) @pytest.mark.parametrize( 'encryption_setting,', ( EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(b'key', 'key-id') ), EncryptionSetting(mode=EncryptionMode.UNKNOWN,), ) ) def test_update_bucket_wrong_encryption(self, encryption_setting): with pytest.raises(WrongEncryptionModeForBucketDefault): self.raw_api.update_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', default_server_side_encryption=encryption_setting, bucket_type='allPublic', ) class TestCreateBucket(BucketTestBase): """Test creating bucket.""" @pytest.mark.parametrize( 'encryption_setting,', ( EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(b'key', 'key-id') ), EncryptionSetting(mode=EncryptionMode.UNKNOWN,), ) ) def test_create_bucket_wrong_encryption(self, encryption_setting): with pytest.raises(WrongEncryptionModeForBucketDefault): self.raw_api.create_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', bucket_type='allPrivate', bucket_info={}, default_server_side_encryption=encryption_setting, ) b2-sdk-python-1.17.3/test/unit/v1/test_scan_policies.py000066400000000000000000000043541426424117700227320ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_scan_policies.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from ..test_base import TestBase from .deps import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from .deps_exception import InvalidArgument class TestScanPolicies(TestBase): def test_default(self): self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_directory('')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_file('')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_directory('a')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_file('a')) def test_exclude_include(self): policy = ScanPoliciesManager(exclude_file_regexes=('a', 'b'), include_file_regexes=('ab',)) self.assertTrue(policy.should_exclude_file('alfa')) self.assertTrue(policy.should_exclude_file('bravo')) self.assertFalse(policy.should_exclude_file('abend')) self.assertFalse(policy.should_exclude_file('charlie')) def test_exclude_dir(self): policy = ScanPoliciesManager(exclude_dir_regexes=('alfa', 'bravo$')) self.assertTrue(policy.should_exclude_directory('alfa')) self.assertTrue(policy.should_exclude_directory('alfa2')) self.assertTrue(policy.should_exclude_directory('alfa/hello')) self.assertTrue(policy.should_exclude_directory('bravo')) self.assertFalse(policy.should_exclude_directory('bravo2')) self.assertFalse(policy.should_exclude_directory('bravo/hello')) self.assertTrue(policy.should_exclude_file('alfa/foo')) self.assertTrue(policy.should_exclude_file('alfa2/hello/foo')) self.assertTrue(policy.should_exclude_file('alfa/hello/foo.txt')) self.assertTrue(policy.should_exclude_file('bravo/foo')) self.assertFalse(policy.should_exclude_file('bravo2/hello/foo')) self.assertTrue(policy.should_exclude_file('bravo/hello/foo.txt')) def test_include_without_exclude(self): with self.assertRaises(InvalidArgument): ScanPoliciesManager(include_file_regexes=('.*',)) b2-sdk-python-1.17.3/test/unit/v1/test_session.py000066400000000000000000000063571426424117700216070ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_session.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import unittest.mock as mock from ..test_base import TestBase from .deps_exception import InvalidAuthToken, Unauthorized from .deps import ALL_CAPABILITIES from .deps import B2Session class TestB2Session(TestBase): def setUp(self): self.account_info = mock.MagicMock() self.account_info.get_account_auth_token.return_value = 'auth_token' self.api = mock.MagicMock() self.api.account_info = self.account_info self.raw_api = mock.MagicMock() self.raw_api.get_file_info_by_id.__name__ = 'get_file_info_by_id' self.raw_api.get_file_info_by_id.side_effect = ['ok'] self.session = B2Session(self.account_info, raw_api=self.raw_api) def test_works_first_time(self): self.assertEqual('ok', self.session.get_file_info_by_id(None)) def test_works_second_time(self): self.raw_api.get_file_info_by_id.side_effect = [ InvalidAuthToken('message', 'code'), 'ok', ] self.assertEqual('ok', self.session.get_file_info_by_id(None)) def test_fails_second_time(self): self.raw_api.get_file_info_by_id.side_effect = [ InvalidAuthToken('message', 'code'), InvalidAuthToken('message', 'code'), ] with self.assertRaises(InvalidAuthToken): self.session.get_file_info_by_id(None) def test_app_key_info_no_info(self): self.account_info.get_allowed.return_value = dict( bucketId=None, bucketName=None, capabilities=ALL_CAPABILITIES, namePrefix=None, ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('no_go', 'code') with self.assertRaisesRegexp( Unauthorized, r'no_go for application key with no restrictions \(code\)' ): self.session.get_file_info_by_id(None) def test_app_key_info_no_info_no_message(self): self.account_info.get_allowed.return_value = dict( bucketId=None, bucketName=None, capabilities=ALL_CAPABILITIES, namePrefix=None, ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('', 'code') with self.assertRaisesRegexp( Unauthorized, r'unauthorized for application key with no restrictions \(code\)' ): self.session.get_file_info_by_id(None) def test_app_key_info_all_info(self): self.account_info.get_allowed.return_value = dict( bucketId='123456', bucketName='my-bucket', capabilities=['readFiles'], namePrefix='prefix/', ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('no_go', 'code') with self.assertRaisesRegexp( Unauthorized, r"no_go for application key with capabilities 'readFiles', restricted to bucket 'my-bucket', restricted to files that start with 'prefix/' \(code\)" ): self.session.get_file_info_by_id(None) b2-sdk-python-1.17.3/test/unit/v1/test_sync.py000066400000000000000000001041461426424117700210730ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_sync.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import concurrent.futures as futures import os import platform import sys import threading import time import unittest from unittest.mock import MagicMock, ANY import pytest from ..test_base import TestBase from .deps import B2Folder, LocalFolder from .deps import BoundedQueueExecutor, zip_folders from .deps import LocalSyncPath, B2SyncPath from .deps import FileVersionInfo from .deps import KeepOrDeleteMode, NewerFileSyncMode, CompareVersionMode from .deps import ScanPoliciesManager, DEFAULT_SCAN_MANAGER from .deps import Synchronizer from .deps import TempDir from .deps import parse_sync_folder from .deps_exception import UnSyncableFilename, NotADirectory, UnableToCreateDirectory, EmptyDirectory, InvalidArgument, CommandError DAY = 86400000 # milliseconds TODAY = DAY * 100 # an arbitrary reference time for testing def write_file(path, contents): parent = os.path.dirname(path) if not os.path.isdir(parent): os.makedirs(parent) with open(path, 'wb') as f: f.write(contents) class TestSync(TestBase): def setUp(self): self.reporter = MagicMock() class TestFolder(TestSync): __test__ = False NAMES = [ '.dot_file', 'hello.', os.path.join('hello', 'a', '1'), os.path.join('hello', 'a', '2'), os.path.join('hello', 'b'), 'hello0', os.path.join('inner', 'a.bin'), os.path.join('inner', 'a.txt'), os.path.join('inner', 'b.bin'), os.path.join('inner', 'b.txt'), os.path.join('inner', 'more', 'a.bin'), os.path.join('inner', 'more', 'a.txt'), '\u81ea\u7531', ] MOD_TIMES = {'.dot_file': TODAY - DAY, 'hello.': TODAY - DAY} def setUp(self): super(TestFolder, self).setUp() self.root_dir = '' def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False, ): raise NotImplementedError def all_files(self, policies_manager): return list(self.prepare_folder().all_files(self.reporter, policies_manager)) def assert_filtered_files(self, scan_results, expected_scan_results): self.assertEqual(expected_scan_results, list(f.relative_path for f in scan_results)) self.reporter.local_access_error.assert_not_called() def test_exclusions(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.txt', 'inner/b.txt', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*\\.bin',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_all(self): expected_list = [] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclusions_inclusions(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager( exclude_file_regexes=('.*\\.bin',), include_file_regexes=('.*a\\.bin',), ) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_matches_prefix(self): expected_list = [ '.dot_file', 'hello.', 'hello/b', 'hello0', 'inner/b.bin', 'inner/b.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*a',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory(self): expected_list = [ '.dot_file', 'hello.', 'hello0', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager( exclude_dir_regexes=('hello', 'more', 'hello0'), exclude_file_regexes=('inner',) ) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory2(self): expected_list = [ '.dot_file', 'hello.', 'hello0', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_dir_regexes=('hello$', 'inner')) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory_trailing_slash_does_not_match(self): expected_list = [ '.dot_file', 'hello.', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_dir_regexes=('hello$', 'inner/')) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclusion_with_exact_match(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('hello0',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_before_in_range(self): expected_list = [ 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_modified_before=TODAY - 100) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_before_exact(self): expected_list = [ 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_modified_before=TODAY) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_after_in_range(self): expected_list = ['.dot_file', 'hello.'] polices_manager = ScanPoliciesManager(exclude_modified_after=TODAY - 100) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_after_exact(self): expected_list = ['.dot_file', 'hello.'] polices_manager = ScanPoliciesManager(exclude_modified_after=TODAY - DAY) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) class TestLocalFolder(TestFolder): __test__ = True def setUp(self): super(TestLocalFolder, self).setUp() self.temp_dir = TempDir() self.root_dir = self.temp_dir.__enter__() def tearDown(self): self.temp_dir.__exit__(*sys.exc_info()) def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False ): assert not (broken_symlink and invalid_permissions) if platform.system() == 'Windows': pytest.skip( 'on Windows there are some environment issues with test directory creation' ) # TODO: fix it if prepare_files: for relative_path in self.NAMES: self.prepare_file(relative_path) if broken_symlink: os.symlink( os.path.join(self.root_dir, 'non_existant_file'), os.path.join(self.root_dir, 'bad_symlink') ) elif invalid_permissions: os.chmod(os.path.join(self.root_dir, self.NAMES[0]), 0) return LocalFolder(self.root_dir) def prepare_file(self, relative_path): path = os.path.join(self.root_dir, relative_path) write_file(path, b'') if relative_path in self.MOD_TIMES: mod_time = int(round(self.MOD_TIMES[relative_path] / 1000)) else: mod_time = int(round(TODAY / 1000)) os.utime(path, (mod_time, mod_time)) def test_slash_sorting(self): # '/' should sort between '.' and '0' folder = self.prepare_folder() self.assertEqual(self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter))) self.reporter.local_access_error.assert_not_called() def test_broken_symlink(self): folder = self.prepare_folder(broken_symlink=True) self.assertEqual(self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter))) self.reporter.local_access_error.assert_called_once_with( os.path.join(self.root_dir, 'bad_symlink') ) def test_invalid_permissions(self): folder = self.prepare_folder(invalid_permissions=True) # tests differ depending on the user running them. "root" will # succeed in os.access(path, os.R_OK) even if the permissions of # the file are 0 as implemented on self._prepare_folder(). # use-case: running test suite inside a docker container if not os.access(os.path.join(self.root_dir, self.NAMES[0]), os.R_OK): self.assertEqual( self.NAMES[1:], list(f.relative_path for f in folder.all_files(self.reporter)) ) self.reporter.local_permission_error.assert_called_once_with( os.path.join(self.root_dir, self.NAMES[0]) ) else: self.assertEqual( self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter)) ) def test_syncable_paths(self): syncable_paths = ( ('test.txt', 'test.txt'), ('./a/test.txt', 'a/test.txt'), ('./a/../test.txt', 'test.txt') ) folder = self.prepare_folder(prepare_files=False) for syncable_path, norm_syncable_path in syncable_paths: expected = os.path.join(self.root_dir, norm_syncable_path.replace('/', os.path.sep)) self.assertEqual(expected, folder.make_full_path(syncable_path)) def test_unsyncable_paths(self): unsyncable_paths = ('../test.txt', 'a/../../test.txt', '/a/test.txt') folder = self.prepare_folder(prepare_files=False) for unsyncable_path in unsyncable_paths: with self.assertRaises(UnSyncableFilename): folder.make_full_path(unsyncable_path) class TestB2Folder(TestFolder): __test__ = True FILE_VERSION_INFOS = { os.path.join('inner', 'a.txt'): [ ( FileVersionInfo( 'a2', 'inner/a.txt', 200, 'text/plain', 'sha1', {}, 2000, 'upload' ), '' ), ( FileVersionInfo( 'a1', 'inner/a.txt', 100, 'text/plain', 'sha1', {}, 1000, 'upload' ), '' ) ], os.path.join('inner', 'b.txt'): [ ( FileVersionInfo( 'b2', 'inner/b.txt', 200, 'text/plain', 'sha1', {}, 1999, 'upload' ), '' ), ( FileVersionInfo( 'bs', 'inner/b.txt', 150, 'text/plain', 'sha1', {}, 1500, 'start' ), '' ), ( FileVersionInfo( 'b1', 'inner/b.txt', 100, 'text/plain', 'sha1', {'src_last_modified_millis': 1001}, 6666, 'upload' ), '' ) ] } def setUp(self): super(TestB2Folder, self).setUp() self.bucket = MagicMock() self.bucket.ls.return_value = [] self.api = MagicMock() self.api.get_bucket_by_name.return_value = self.bucket def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False ): if prepare_files: for relative_path in self.NAMES: self.prepare_file(relative_path, use_file_versions_info) return B2Folder('bucket-name', self.root_dir, self.api) def prepare_file(self, relative_path, use_file_versions_info=False): if use_file_versions_info and relative_path in self.FILE_VERSION_INFOS: self.bucket.ls.return_value.extend(self.FILE_VERSION_INFOS[relative_path]) return if platform.system() == 'Windows': relative_path = relative_path.replace(os.sep, '/') if relative_path in self.MOD_TIMES: self.bucket.ls.return_value.append( ( FileVersionInfo( relative_path, relative_path, 100, 'text/plain', 'sha1', {}, self.MOD_TIMES[relative_path], 'upload' ), self.root_dir ) ) else: self.bucket.ls.return_value.append( ( FileVersionInfo( relative_path, relative_path, 100, 'text/plain', 'sha1', {}, TODAY, 'upload' ), self.root_dir ) ) def test_empty(self): folder = self.prepare_folder(prepare_files=False) self.assertEqual([], list(folder.all_files(self.reporter))) def test_multiple_versions(self): # Test two files, to cover the yield within the loop, and the yield without. folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( [ "B2Path(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", "B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])" ], [ str(f) for f in folder.all_files(self.reporter) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ] ) def test_exclude_modified_multiple_versions(self): polices_manager = ScanPoliciesManager( exclude_modified_before=1001, exclude_modified_after=1999 ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( ["B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ str(f) for f in folder.all_files(self.reporter, policies_manager=polices_manager) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ] ) def test_exclude_modified_all_versions(self): polices_manager = ScanPoliciesManager( exclude_modified_before=1500, exclude_modified_after=1500 ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( [], list(folder.all_files(self.reporter, policies_manager=polices_manager)) ) # Path names not allowed to be sync'd on Windows NOT_SYNCD_ON_WINDOWS = [ 'Z:/windows/system32/drivers/etc/hosts', 'a:/Users/.default/test', r'C:\Windows\system32\drivers\mstsc.sys', ] def test_unsyncable_filenames(self): b2_folder = B2Folder('bucket-name', '', self.api) # Test a list of unsyncable file names filenames_to_test = [ '/', # absolute root path '//', '///', '/..', '/../', '/.../', '/../.', '/../..', '/a.txt', '/folder/a.txt', './folder/a.txt', # current dir relative path 'folder/./a.txt', 'folder/folder/.', 'a//b/', # double-slashes 'a///b', 'a////b', '../test', # start with parent dir '../../test', '../../abc/../test', '../../abc/../test/', '../../abc/../.test', 'a/b/c/../d', # parent dir embedded 'a//..//b../..c/', '..a/b../..c/../d..', 'a/../', 'a/../../../../../', 'a/b/c/..', r'\\', # backslash filenames r'\z', r'..\\', r'..\..', r'\..\\', r'\\..\\..', r'\\', r'\\\\', r'\\\\server\\share\\dir\\file', r'\\server\share\dir\file', r'\\?\C\Drive\temp', r'.\\//', r'..\\..//..\\\\', r'.\\a\\..\\b', r'a\\.\\b', ] if platform.system() == "Windows": filenames_to_test.extend(self.NOT_SYNCD_ON_WINDOWS) for filename in filenames_to_test: self.bucket.ls.return_value = [ (FileVersionInfo('a1', filename, 1, 'text/plain', 'sha1', {}, 1000, 'upload'), '') ] try: list(b2_folder.all_files(self.reporter)) self.fail("should have thrown UnSyncableFilename for: '%s'" % filename) except UnSyncableFilename as e: self.assertTrue(filename in str(e)) def test_syncable_filenames(self): b2_folder = B2Folder('bucket-name', '', self.api) # Test a list of syncable file names filenames_to_test = [ '', ' ', ' / ', ' ./. ', ' ../.. ', '.. / ..', r'.. \ ..', 'file.txt', '.folder/', '..folder/', '..file', r'file/ and\ folder', 'file..', '..file..', 'folder/a.txt..', '..a/b../c../..d/e../', r'folder\test', r'folder\..f..\..f\..f', r'mix/and\match/', r'a\b\c\d', ] # filenames not permitted on Windows *should* be allowed on Linux if platform.system() != "Windows": filenames_to_test.extend(self.NOT_SYNCD_ON_WINDOWS) for filename in filenames_to_test: self.bucket.ls.return_value = [ (FileVersionInfo('a1', filename, 1, 'text/plain', 'sha1', {}, 1000, 'upload'), '') ] list(b2_folder.all_files(self.reporter)) class FakeLocalFolder(LocalFolder): def __init__(self, local_sync_paths): super().__init__('folder') self.local_sync_paths = local_sync_paths def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): for single_path in self.local_sync_paths: if single_path.relative_path.endswith('/'): if policies_manager.should_exclude_b2_directory(single_path.relative_path): continue else: if policies_manager.should_exclude_local_path(single_path): continue yield single_path def make_full_path(self, name): return '/dir/' + name class FakeB2Folder(B2Folder): def __init__(self, test_files): self.file_versions = [] for test_file in test_files: self.file_versions.extend(self.file_versions_from_file_tuples(*test_file)) super().__init__('test-bucket', 'folder', MagicMock()) def get_file_versions(self): yield from iter(self.file_versions) @classmethod def file_versions_from_file_tuples(cls, name, mod_times, size=10): """ Makes FileVersion objects. Positive modification times are uploads, and negative modification times are hides. It's a hack, but it works. """ try: mod_times = iter(mod_times) except TypeError: mod_times = [mod_times] return [ FileVersionInfo( id_='id_%s_%d' % (name[0], abs(mod_time)), file_name='folder/' + name, upload_timestamp=abs(mod_time), action='upload' if 0 < mod_time else 'hide', size=size, file_info={'in_b2': 'yes'}, content_type='text/plain', content_sha1='content_sha1', ) for mod_time in mod_times ] # yapf disable @classmethod def sync_path_from_file_tuple(cls, name, mod_times, size=10): file_versions = cls.file_versions_from_file_tuples(name, mod_times, size) return B2SyncPath(name, file_versions[0], file_versions) class TestParseSyncFolder(TestBase): def test_b2_double_slash(self): self._check_one('B2Folder(my-bucket, folder/path)', 'b2://my-bucket/folder/path') def test_b2_no_double_slash(self): self._check_one('B2Folder(my-bucket, folder/path)', 'b2:my-bucket/folder/path') def test_b2_trailing_slash(self): self._check_one('B2Folder(my-bucket, a)', 'b2://my-bucket/a/') def test_b2_no_folder(self): self._check_one('B2Folder(my-bucket, )', 'b2://my-bucket') self._check_one('B2Folder(my-bucket, )', 'b2://my-bucket/') def test_local(self): if platform.system() == 'Windows': drive, _ = os.path.splitdrive(os.getcwd()) expected = 'LocalFolder(\\\\?\\%s\\foo)' % (drive,) else: expected = 'LocalFolder(/foo)' self._check_one(expected, '/foo') def test_local_trailing_slash(self): if platform.system() == 'Windows': drive, _ = os.path.splitdrive(os.getcwd()) expected = 'LocalFolder(\\\\?\\%s\\foo)' % (drive,) else: expected = 'LocalFolder(/foo)' self._check_one(expected, '/foo/') def _check_one(self, expected, to_parse): api = MagicMock() self.assertEqual(expected, str(parse_sync_folder(str(to_parse), api))) class TestFolderExceptions: """There is an exact copy of this class in unit/v0/test_sync.py - TODO: leave only one when migrating tests to sync-like structure. """ @pytest.mark.parametrize( 'exception,msg', [ pytest.param(NotADirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param(Exception, 'is not a directory', marks=pytest.mark.apiver(to_ver=1)), ], ) def test_ensure_present_not_a_dir(self, exception, msg): with TempDir() as path: file = os.path.join(path, 'clearly_a_file') with open(file, 'w') as f: f.write(' ') folder = parse_sync_folder(file, MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_present() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(UnableToCreateDirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( Exception, 'unable to create directory', marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_ensure_present_unable_to_create(self, exception, msg): with TempDir() as path: file = os.path.join(path, 'clearly_a_file') with open(file, 'w') as f: f.write(' ') folder = parse_sync_folder(os.path.join(file, 'nonsense'), MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_present() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(EmptyDirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( CommandError, 'Directory .* is empty. Use --allowEmptySource to sync anyway.', marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_ensure_non_empty(self, exception, msg): with TempDir() as path: folder = parse_sync_folder(path, MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_non_empty() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(InvalidArgument, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( CommandError, "'//' not allowed in path names", marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_double_slash_not_allowed(self, exception, msg): with pytest.raises(exception, match=msg): parse_sync_folder('b2://a//b', MagicMock()) class TestZipFolders(TestSync): def test_empty(self): folder_a = FakeB2Folder([]) folder_b = FakeB2Folder([]) self.assertEqual([], list(zip_folders(folder_a, folder_b, self.reporter))) def test_one_empty(self): file_a1 = LocalSyncPath("a.txt", "a.txt", 100, 10) folder_a = FakeLocalFolder([file_a1]) folder_b = FakeB2Folder([]) self.assertEqual([(file_a1, None)], list(zip_folders(folder_a, folder_b, self.reporter))) def test_two(self): file_a1 = ("a.txt", [100], 10) file_a2 = ("b.txt", [100], 10) file_a3 = ("d.txt", [100], 10) file_a4 = ("f.txt", [100], 10) file_b1 = ("b.txt", [200], 10) file_b2 = ("e.txt", [200], 10) folder_a = FakeB2Folder([file_a1, file_a2, file_a3, file_a4]) folder_b = FakeB2Folder([file_b1, file_b2]) self.assertEqual( [ (FakeB2Folder.sync_path_from_file_tuple(*file_a1), None), ( FakeB2Folder.sync_path_from_file_tuple(*file_a2), FakeB2Folder.sync_path_from_file_tuple(*file_b1) ), (FakeB2Folder.sync_path_from_file_tuple(*file_a3), None), (None, FakeB2Folder.sync_path_from_file_tuple(*file_b2)), (FakeB2Folder.sync_path_from_file_tuple(*file_a4), None) ], list(zip_folders(folder_a, folder_b, self.reporter)) ) def test_pass_reporter_to_folder(self): """ Check that the zip_folders() function passes the reporter through to both folders. """ folder_a = MagicMock() folder_b = MagicMock() folder_a.all_files = MagicMock(return_value=iter([])) folder_b.all_files = MagicMock(return_value=iter([])) self.assertEqual([], list(zip_folders(folder_a, folder_b, self.reporter))) folder_a.all_files.assert_called_once_with(self.reporter, ANY) folder_b.all_files.assert_called_once_with(self.reporter) class FakeArgs: """ Can be passed to sync code to simulate command-line options. """ def __init__( self, newer_file_mode=NewerFileSyncMode.RAISE_ERROR, keep_days_or_delete=KeepOrDeleteMode.NO_DELETE, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=None, keep_days=None, excludeRegex=None, excludeDirRegex=None, includeRegex=None, debugLogs=True, dryRun=False, allowEmptySource=False, excludeAllSymlinks=False, ): self.newer_file_mode = newer_file_mode self.keep_days_or_delete = keep_days_or_delete self.keep_days = keep_days self.compare_version_mode = compare_version_mode self.compare_threshold = compare_threshold if excludeRegex is None: excludeRegex = [] self.excludeRegex = excludeRegex if includeRegex is None: includeRegex = [] self.includeRegex = includeRegex if excludeDirRegex is None: excludeDirRegex = [] self.excludeDirRegex = excludeDirRegex self.debugLogs = debugLogs self.dryRun = dryRun self.allowEmptySource = allowEmptySource self.excludeAllSymlinks = excludeAllSymlinks def get_synchronizer(self, policies_manager=DEFAULT_SCAN_MANAGER): return Synchronizer( 1, policies_manager=policies_manager, dry_run=self.dryRun, allow_empty_source=self.allowEmptySource, newer_file_mode=self.newer_file_mode, keep_days_or_delete=self.keep_days_or_delete, keep_days=self.keep_days, compare_version_mode=self.compare_version_mode, compare_threshold=self.compare_threshold, ) def local_file(name, mod_time, size=10): """ Makes a LocalSyncPath object for a b2 file. """ return LocalSyncPath(name, name, mod_time, size) class TestExclusions(TestSync): def _check_folder_sync(self, expected_actions, fakeargs): # only local file_a = ('a.txt', 100) file_b = ('b.txt', 100) file_d = ('d/d.txt', 100) file_e = ('e/e.incl', 100) # both local and remote file_bi = ('b.txt.incl', 100) file_z = ('z.incl', 100) # only remote file_c = ('c.txt', 100) local_folder = FakeLocalFolder( [local_file(*f) for f in (file_a, file_b, file_d, file_e, file_bi, file_z)] ) b2_folder = FakeB2Folder([file_bi, file_c, file_z]) policies_manager = ScanPoliciesManager( exclude_dir_regexes=fakeargs.excludeDirRegex, exclude_file_regexes=fakeargs.excludeRegex, include_file_regexes=fakeargs.includeRegex, exclude_all_symlinks=fakeargs.excludeAllSymlinks ) synchronizer = fakeargs.get_synchronizer(policies_manager=policies_manager) actions = list( synchronizer.make_folder_sync_actions( local_folder, b2_folder, TODAY, self.reporter, policies_manager ) ) self.assertEqual(expected_actions, [str(a) for a in actions]) def test_file_exclusions_with_delete(self): expected_actions = [ 'b2_upload(/dir/a.txt, folder/a.txt, 100)', 'b2_delete(folder/b.txt.incl, id_b_100, )', 'b2_delete(folder/c.txt, id_c_100, )', 'b2_upload(/dir/d/d.txt, folder/d/d.txt, 100)', 'b2_upload(/dir/e/e.incl, folder/e/e.incl, 100)', ] self._check_folder_sync( expected_actions, FakeArgs( keep_days_or_delete=KeepOrDeleteMode.DELETE, excludeRegex=["b\\.txt"], ), ) def test_file_exclusions_inclusions_with_delete(self): expected_actions = [ 'b2_upload(/dir/a.txt, folder/a.txt, 100)', 'b2_delete(folder/b.txt.incl, id_b_100, )', 'b2_delete(folder/c.txt, id_c_100, )', 'b2_upload(/dir/d/d.txt, folder/d/d.txt, 100)', 'b2_upload(/dir/e/e.incl, folder/e/e.incl, 100)', 'b2_upload(/dir/b.txt.incl, folder/b.txt.incl, 100)', ] fakeargs = FakeArgs( keep_days_or_delete=KeepOrDeleteMode.DELETE, excludeRegex=["b\\.txt"], includeRegex=[".*\\.incl"] ) self._check_folder_sync(expected_actions, fakeargs) class TestBoundedQueueExecutor(TestBase): def test_run_more_than_queue_size(self): """ Makes sure that the executor will run more jobs that the queue size, which ensures that the semaphore gets released, even if an exception is thrown. """ raw_executor = futures.ThreadPoolExecutor(1) bounded_executor = BoundedQueueExecutor(raw_executor, 5) class Counter: """ Counts how many times run() is called. """ def __init__(self): self.counter = 0 def run(self): """ Always increments the counter. Sometimes raises an exception. """ self.counter += 1 if self.counter % 2 == 0: raise Exception('test') counter = Counter() for _ in range(10): bounded_executor.submit(counter.run) bounded_executor.shutdown() self.assertEqual(10, counter.counter) def test_wait_for_running_jobs(self): """ Makes sure that no more than queue_limit workers are running at once, which checks that the semaphore is acquired before submitting an action. """ raw_executor = futures.ThreadPoolExecutor(2) bounded_executor = BoundedQueueExecutor(raw_executor, 1) assert_equal = self.assertEqual class CountAtOnce: """ Counts how many threads are running at once. There should never be more than 1 because that's the limit on the bounded executor. """ def __init__(self): self.running_at_once = 0 self.lock = threading.Lock() def run(self): with self.lock: self.running_at_once += 1 assert_equal(1, self.running_at_once) # While we are sleeping here, no other actions should start # running. If they do, they will increment the counter and # fail the above assertion. time.sleep(0.05) with self.lock: self.running_at_once -= 1 self.counter += 1 if self.counter % 2 == 0: raise Exception('test') count_at_once = CountAtOnce() for _ in range(5): bounded_executor.submit(count_at_once.run) bounded_executor.shutdown() if __name__ == '__main__': unittest.main() b2-sdk-python-1.17.3/test/unit/v1/test_utils.py000066400000000000000000000154721426424117700212620ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_utils.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from ..test_base import TestBase from .deps import b2_url_encode, b2_url_decode, choose_part_ranges, format_and_scale_number, format_and_scale_fraction # These are from the B2 Docs (https://www.backblaze.com/b2/docs/string_encoding.html) ENCODING_TEST_CASES = [ { 'fullyEncoded': '%20', 'minimallyEncoded': '+', 'string': ' ' }, { 'fullyEncoded': '%21', 'minimallyEncoded': '!', 'string': '!' }, { 'fullyEncoded': '%22', 'minimallyEncoded': '%22', 'string': '"' }, { 'fullyEncoded': '%23', 'minimallyEncoded': '%23', 'string': '#' }, { 'fullyEncoded': '%24', 'minimallyEncoded': '$', 'string': '$' }, { 'fullyEncoded': '%25', 'minimallyEncoded': '%25', 'string': '%' }, { 'fullyEncoded': '%26', 'minimallyEncoded': '%26', 'string': '&' }, { 'fullyEncoded': '%27', 'minimallyEncoded': "'", 'string': "'" }, { 'fullyEncoded': '%28', 'minimallyEncoded': '(', 'string': '(' }, { 'fullyEncoded': '%29', 'minimallyEncoded': ')', 'string': ')' }, { 'fullyEncoded': '%2A', 'minimallyEncoded': '*', 'string': '*' }, { 'fullyEncoded': '%2B', 'minimallyEncoded': '%2B', 'string': '+' }, { 'fullyEncoded': '%2C', 'minimallyEncoded': '%2C', 'string': ',' }, { 'fullyEncoded': '%2D', 'minimallyEncoded': '-', 'string': '-' }, { 'fullyEncoded': '%2E', 'minimallyEncoded': '.', 'string': '.' }, { 'fullyEncoded': '/', 'minimallyEncoded': '/', 'string': '/' }, { 'fullyEncoded': '%30', 'minimallyEncoded': '0', 'string': '0' }, { 'fullyEncoded': '%39', 'minimallyEncoded': '9', 'string': '9' }, { 'fullyEncoded': '%3A', 'minimallyEncoded': ':', 'string': ':' }, { 'fullyEncoded': '%3B', 'minimallyEncoded': ';', 'string': ';' }, { 'fullyEncoded': '%3C', 'minimallyEncoded': '%3C', 'string': '<' }, { 'fullyEncoded': '%3D', 'minimallyEncoded': '=', 'string': '=' }, { 'fullyEncoded': '%3E', 'minimallyEncoded': '%3E', 'string': '>' }, { 'fullyEncoded': '%3F', 'minimallyEncoded': '%3F', 'string': '?' }, { 'fullyEncoded': '%40', 'minimallyEncoded': '@', 'string': '@' }, { 'fullyEncoded': '%41', 'minimallyEncoded': 'A', 'string': 'A' }, { 'fullyEncoded': '%5A', 'minimallyEncoded': 'Z', 'string': 'Z' }, { 'fullyEncoded': '%5B', 'minimallyEncoded': '%5B', 'string': '[' }, { 'fullyEncoded': '%5C', 'minimallyEncoded': '%5C', 'string': '\\' }, { 'fullyEncoded': '%5D', 'minimallyEncoded': '%5D', 'string': ']' }, { 'fullyEncoded': '%5E', 'minimallyEncoded': '%5E', 'string': '^' }, { 'fullyEncoded': '%5F', 'minimallyEncoded': '_', 'string': '_' }, { 'fullyEncoded': '%60', 'minimallyEncoded': '%60', 'string': '`' }, { 'fullyEncoded': '%61', 'minimallyEncoded': 'a', 'string': 'a' }, { 'fullyEncoded': '%7A', 'minimallyEncoded': 'z', 'string': 'z' }, { 'fullyEncoded': '%7B', 'minimallyEncoded': '%7B', 'string': '{' }, { 'fullyEncoded': '%7C', 'minimallyEncoded': '%7C', 'string': '|' }, { 'fullyEncoded': '%7D', 'minimallyEncoded': '%7D', 'string': '}' }, { 'fullyEncoded': '%7E', 'minimallyEncoded': '~', 'string': '~' }, { 'fullyEncoded': '%7F', 'minimallyEncoded': '%7F', 'string': u'\u007f' }, { 'fullyEncoded': '%E8%87%AA%E7%94%B1', 'minimallyEncoded': '%E8%87%AA%E7%94%B1', 'string': u'\u81ea\u7531' }, { 'fullyEncoded': '%F0%90%90%80', 'minimallyEncoded': '%F0%90%90%80', 'string': u'\U00010400' }, ] class TestUrlEncoding(TestBase): def test_it(self): for test_case in ENCODING_TEST_CASES: string = test_case['string'] fully_encoded = test_case['fullyEncoded'] minimally_encoded = test_case['minimallyEncoded'] encoded = b2_url_encode(string) expected_encoded = (minimally_encoded, fully_encoded) if encoded not in expected_encoded: print( 'string: %s encoded: %s expected: %s' % (repr(string), encoded, expected_encoded) ) self.assertTrue(encoded in expected_encoded) self.assertEqual(string, b2_url_decode(fully_encoded)) self.assertEqual(string, b2_url_decode(minimally_encoded)) class TestChooseParts(TestBase): def test_it(self): self._check_one([(0, 100), (100, 100)], 200, 100) self._check_one([(0, 149), (149, 150)], 299, 100) self._check_one([(0, 100), (100, 100), (200, 100)], 300, 100) ten_TB = 10 * 1000 * 1000 * 1000 * 1000 one_GB = 1000 * 1000 * 1000 expected = [(i * one_GB, one_GB) for i in range(10000)] actual = choose_part_ranges(ten_TB, 100 * 1000 * 1000) self.assertEqual(expected, actual) def _check_one(self, expected, content_length, min_part_size): self.assertEqual(expected, choose_part_ranges(content_length, min_part_size)) class TestFormatAndScaleNumber(TestBase): def test_it(self): self._check_one('1 B', 1) self._check_one('999 B', 999) self._check_one('1.00 kB', 1000) self._check_one('999 kB', 999000) def _check_one(self, expected, x): self.assertEqual(expected, format_and_scale_number(x, 'B')) class TestFormatAndScaleFraction(TestBase): def test_it(self): self._check_one('0 / 100 B', 0, 100) self._check_one('0.0 / 10.0 kB', 0, 10000) self._check_one('9.4 / 10.0 kB', 9400, 10000) def _check_one(self, expected, numerator, denominator): self.assertEqual(expected, format_and_scale_fraction(numerator, denominator, 'B')) b2-sdk-python-1.17.3/test/unit/v1/test_version_utils.py000066400000000000000000000077401426424117700230260ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_version_utils.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import warnings from ..test_base import TestBase from .deps import rename_argument, rename_function class TestRenameArgument(TestBase): VERSION = '0.1.10' def test_warning(self): @rename_argument('aaa', 'bbb', '0.1.0', '0.2.0', current_version=self.VERSION) def easy(bbb): """ easy docstring """ return bbb # check that warning is not emitted too early with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") assert easy(5) == 5 assert easy(bbb=5) == 5 assert easy.__name__ == 'easy' assert easy.__doc__ == ' easy docstring ' assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") assert easy(aaa=5) == 5 assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert str( w[-1].message ) == "'aaa' is a deprecated argument for 'easy' function/method - it was renamed to 'bbb' in version 0.1.0. Support for the old name is going to be dropped in 0.2.0.", str( w[-1].message ) def test_outdated_replacement(self): with self.assertRaises( AssertionError, msg= "rename_argument decorator is still used in version %s when old argument name 'aaa' was scheduled to be dropped in 0.1.2. It is time to remove the mapping." % (self.VERSION,), ): @rename_argument('aaa', 'bbb', '0.1.0', '0.1.2', current_version=self.VERSION) def late(bbb): return bbb assert late # make linters happy def test_future_replacement(self): with self.assertRaises( AssertionError, msg= "rename_argument decorator indicates that the replacement of argument 'aaa' should take place in the future version 0.2.0, while the current version is 0.2.2. It looks like should be _discouraged_ at this point and not _deprecated_ yet. Consider using 'discourage_argument' decorator instead." ): @rename_argument('aaa', 'bbb', '0.2.0', '0.2.2', current_version=self.VERSION) def early(bbb): return bbb assert early # make linters happy def test_inverted_versions(self): with self.assertRaises( AssertionError, msg= "rename_argument decorator is set to start renaming argument 'aaa' starting at version 0.2.2 and finishing in 0.2.0. It needs to start at a lower version and finish at a higher version." ): @rename_argument('aaa', 'bbb', '0.2.2', '0.2.0', current_version=self.VERSION) def backwards(bbb): return bbb assert backwards # make linters happy class TestRenameFunction(TestBase): VERSION = '0.1.10' def test_rename_function(self): def new(bbb): return bbb for i in ('new', new): @rename_function(i, '0.1.0', '0.2.0', current_version=self.VERSION) def old(bbb): return bbb with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") assert old(5) == 5 assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert str( w[-1].message ) == "'old' is deprecated since version 0.1.0 - it was moved to 'new', please switch to use that. The proxy for the old name is going to be removed in 0.2.0.", str( w[-1].message ) b2-sdk-python-1.17.3/test/unit/v2/000077500000000000000000000000001426424117700165015ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/v2/__init__.py000066400000000000000000000004431426424117700206130ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/v2/apiver/000077500000000000000000000000001426424117700177675ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/v2/apiver/__init__.py000066400000000000000000000006221426424117700221000ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/apiver/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details b2-sdk-python-1.17.3/test/unit/v2/apiver/apiver_deps.py000066400000000000000000000005241426424117700226430ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/apiver/apiver_deps.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.v2 import * # noqa V = 2 b2-sdk-python-1.17.3/test/unit/v2/apiver/apiver_deps_exception.py000066400000000000000000000005411426424117700247200ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/apiver/apiver_deps_exception.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk.v2.exception import * # noqa b2-sdk-python-1.17.3/test/unit/v2/test_transfer.py000066400000000000000000000016751426424117700217470ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/test_transfer.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from unittest.mock import Mock from ..test_base import TestBase from .apiver.apiver_deps import DownloadManager, UploadManager class TestDownloadManager(TestBase): def test_set_thread_pool_size(self) -> None: download_manager = DownloadManager(services=Mock()) download_manager.set_thread_pool_size(21) self.assertEqual(download_manager._thread_pool._max_workers, 21) class TestUploadManager(TestBase): def test_set_thread_pool_size(self) -> None: upload_manager = UploadManager(services=Mock()) upload_manager.set_thread_pool_size(37) self.assertEqual(upload_manager._thread_pool._max_workers, 37) b2-sdk-python-1.17.3/test/unit/v3/000077500000000000000000000000001426424117700165025ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/v3/__init__.py000066400000000000000000000004431426424117700206140ustar00rootroot00000000000000###################################################################### # # File: test/unit/v3/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/v3/apiver/000077500000000000000000000000001426424117700177705ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/v3/apiver/__init__.py000066400000000000000000000006221426424117700221010ustar00rootroot00000000000000###################################################################### # # File: test/unit/v3/apiver/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details b2-sdk-python-1.17.3/test/unit/v3/apiver/apiver_deps.py000066400000000000000000000005251426424117700226450ustar00rootroot00000000000000###################################################################### # # File: test/unit/v3/apiver/apiver_deps.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk._v3 import * # noqa V = 3 b2-sdk-python-1.17.3/test/unit/v3/apiver/apiver_deps_exception.py000066400000000000000000000005421426424117700247220ustar00rootroot00000000000000###################################################################### # # File: test/unit/v3/apiver/apiver_deps_exception.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk._v3.exception import * # noqa b2-sdk-python-1.17.3/test/unit/v_all/000077500000000000000000000000001426424117700172475ustar00rootroot00000000000000b2-sdk-python-1.17.3/test/unit/v_all/__init__.py000066400000000000000000000004461426424117700213640ustar00rootroot00000000000000###################################################################### # # File: test/unit/v_all/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-1.17.3/test/unit/v_all/test_api.py000066400000000000000000000060411426424117700214320ustar00rootroot00000000000000###################################################################### # # File: test/unit/v_all/test_api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from apiver_deps import B2Api from apiver_deps import B2HttpApiConfig from apiver_deps import Bucket from apiver_deps import InMemoryCache from apiver_deps import EncryptionMode from apiver_deps import EncryptionSetting from apiver_deps import InMemoryAccountInfo from apiver_deps import RawSimulator from apiver_deps_exception import BucketIdNotFound from ..test_base import TestBase class TestApi(TestBase): def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = InMemoryCache() self.api = B2Api( self.account_info, self.cache, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator) ) self.raw_api = self.api.session.raw_api (self.application_key_id, self.master_key) = self.raw_api.create_account() def _authorize_account(self): self.api.authorize_account('production', self.application_key_id, self.master_key) @pytest.mark.apiver(to_ver=1) def test_get_bucket_by_id_up_to_v1(self): bucket = self.api.get_bucket_by_id("this id doesn't even exist") assert bucket.id_ == "this id doesn't even exist" for att_name, att_value in [ ('name', None), ('type_', None), ('bucket_info', {}), ('cors_rules', []), ('lifecycle_rules', []), ('revision', None), ('bucket_dict', {}), ('options_set', set()), ('default_server_side_encryption', EncryptionSetting(EncryptionMode.UNKNOWN)), ]: with self.subTest(att_name=att_name): assert getattr(bucket, att_name) == att_value, att_name @pytest.mark.apiver(from_ver=2) def test_get_bucket_by_id_v2(self): self._authorize_account() with pytest.raises(BucketIdNotFound): self.api.get_bucket_by_id("this id doesn't even exist") created_bucket = self.api.create_bucket('bucket1', 'allPrivate') read_bucket = self.api.get_bucket_by_id(created_bucket.id_) assert created_bucket.id_ == read_bucket.id_ self.cache.save_bucket(Bucket(api=self.api, name='bucket_name', id_='bucket_id')) read_bucket = self.api.get_bucket_by_id('bucket_id') assert read_bucket.name == 'bucket_name' def test_get_download_url_for_file_name(self): self._authorize_account() download_url = self.api.get_download_url_for_file_name('bucket1', 'some-file.txt') assert download_url == 'http://download.example.com/file/bucket1/some-file.txt' def test_get_download_url_for_fileid(self): self._authorize_account() download_url = self.api.get_download_url_for_fileid('file-id') assert download_url == 'http://download.example.com/b2api/v2/b2_download_file_by_id?fileId=file-id' b2-sdk-python-1.17.3/test/unit/v_all/test_replication.py000066400000000000000000000115071426424117700231750ustar00rootroot00000000000000###################################################################### # # File: test/unit/v_all/test_replication.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import logging import pytest from apiver_deps import B2Api from apiver_deps import B2HttpApiConfig from apiver_deps import InMemoryCache from apiver_deps import InMemoryAccountInfo from apiver_deps import RawSimulator from apiver_deps import ReplicationConfiguration, ReplicationRule, ReplicationSetupHelper from ..test_base import TestBase logger = logging.getLogger(__name__) class TestReplication(TestBase): def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = InMemoryCache() self.api = B2Api( self.account_info, self.cache, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator) ) self.raw_api = self.api.session.raw_api self.application_key_id, self.master_key = self.raw_api.create_account() def _authorize_account(self): self.api.authorize_account('production', self.application_key_id, self.master_key) @pytest.mark.apiver(from_ver=2) def test_setup_both(self): self._authorize_account() source_bucket = self.api.create_bucket('bucket1', 'allPrivate') destination_bucket = self.api.create_bucket('bucket2', 'allPrivate') logger.info('preparations complete, starting the test') rsh = ReplicationSetupHelper() source_bucket, destination_bucket = rsh.setup_both( source_bucket=source_bucket, destination_bucket=destination_bucket, name='aa', prefix='ab', ) from pprint import pprint pprint([k.as_dict() for k in self.api.list_keys()]) keymap = {k.key_name: k for k in self.api.list_keys()} source_application_key = keymap['bucket1-replisrc'] assert source_application_key assert set(source_application_key.capabilities) == set( ('readFiles', 'readFileLegalHolds', 'readFileRetentions') ) assert not source_application_key.name_prefix assert source_application_key.expiration_timestamp_millis is None destination_application_key = keymap['bucket2-replidst'] assert destination_application_key assert set(destination_application_key.capabilities) == set( ('writeFiles', 'writeFileLegalHolds', 'writeFileRetentions', 'deleteFiles') ) assert not destination_application_key.name_prefix assert destination_application_key.expiration_timestamp_millis is None assert source_bucket.replication.rules == [ ReplicationRule( destination_bucket_id='bucket_1', name='aa', file_name_prefix='ab', is_enabled=True, priority=128, ) ] assert source_bucket.replication.source_key_id == source_application_key.id_ assert source_bucket.replication.source_to_destination_key_mapping == {} print(destination_bucket.replication) assert destination_bucket.replication.rules == [] assert destination_bucket.replication.source_key_id is None assert destination_bucket.replication.source_to_destination_key_mapping == { source_application_key.id_: destination_application_key.id_ } old_source_application_key = source_application_key source_bucket, destination_bucket = rsh.setup_both( source_bucket=source_bucket, destination_bucket=destination_bucket, prefix='ad', include_existing_files=True, ) keymap = {k.key_name: k for k in self.api.list_keys()} new_source_application_key = keymap['bucket1-replisrc'] assert source_bucket.replication.rules == [ ReplicationRule( destination_bucket_id='bucket_1', name='aa', file_name_prefix='ab', is_enabled=True, priority=128, ), ReplicationRule( destination_bucket_id='bucket_1', name='bucket2', file_name_prefix='ad', is_enabled=True, priority=133, include_existing_files=True, ), ] assert source_bucket.replication.source_key_id == old_source_application_key.id_ assert destination_bucket.replication.source_to_destination_key_mapping == { new_source_application_key.id_: destination_application_key.id_ } @pytest.mark.apiver(from_ver=2) def test_factory(self): replication = ReplicationConfiguration.from_dict({}) assert replication == ReplicationConfiguration()