pax_global_header00006660000000000000000000000064147445437000014522gustar00rootroot0000000000000052 comment=c214f59102c3cc26c9fe4502f5249272de03ae40 b2-sdk-python-2.8.0/000077500000000000000000000000001474454370000141325ustar00rootroot00000000000000b2-sdk-python-2.8.0/.github/000077500000000000000000000000001474454370000154725ustar00rootroot00000000000000b2-sdk-python-2.8.0/.github/SUPPORT.md000066400000000000000000000012711474454370000171710ustar00rootroot00000000000000Issues with **B2_Command_line_tool** (problems with the tool itself) should be reported at [B2 CLI issue tracker](https://github.com/Backblaze/B2_Command_Line_Tool/issues). Issuew with **B2 cloud service** (not caused by B2 CLI or sdk) should be reported directly to [Backblaze support](https://help.backblaze.com/hc/en-us/requests/new). Issues with the B2 python sdk should be reported in [b2-sdk-python issue tracker](https://github.com/Backblaze/b2-sdk-python/issues). This should be used by authors of tools that interact with the b2 cloud through **b2-sdk-python**, but also if an issue with some other project (that uses **b2-sdk-python**) is clearly caused by a bug in **b2-sdk-python**. b2-sdk-python-2.8.0/.github/dependabot.yml000066400000000000000000000005041474454370000203210ustar00rootroot00000000000000# Documentation available at # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: - package-ecosystem: "pip" directory: "/" schedule: interval: "daily" # This setting does not affect security updates open-pull-requests-limit: 0b2-sdk-python-2.8.0/.github/no-response.yml000066400000000000000000000013211474454370000204620ustar00rootroot00000000000000# Configuration for probot-no-response - https://github.com/probot/no-response # Number of days of inactivity before an Issue is closed for lack of response daysUntilClose: 14 # Label requiring a response responseRequiredLabel: more-information-needed # Comment to post when closing an Issue for lack of response. Set to `false` to disable closeComment: > This issue has been automatically closed because there has been no response to our request for more information from the original author. With only the information that is currently in the issue, we don't have enough information to take action. Please reach out if you have or find the answers we need so that we can investigate or assist you further. b2-sdk-python-2.8.0/.github/workflows/000077500000000000000000000000001474454370000175275ustar00rootroot00000000000000b2-sdk-python-2.8.0/.github/workflows/cd.yml000066400000000000000000000036721474454370000206500ustar00rootroot00000000000000name: Continuous Delivery on: push: tags: 'v*' # push events to matching v*, i.e. v1.0, v20.15.10 env: PYTHON_DEFAULT_VERSION: "3.12" jobs: deploy: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} B2_PYPI_PASSWORD: ${{ secrets.B2_PYPI_PASSWORD }} runs-on: ubuntu-latest steps: - name: Determine if pre-release id: prerelease_check run: | export IS_PRERELEASE=$([[ ${{ github.ref }} =~ [^0-9]$ ]] && echo true || echo false) echo "prerelease=$IS_PRERELEASE" >> $GITHUB_OUTPUT - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - name: Display Python version run: python -c "import sys; print(sys.version)" - name: Install dependencies run: python -m pip install --upgrade nox pdm - name: Build the distribution id: build run: nox -vs build - name: Read the Changelog id: read-changelog uses: mindsers/changelog-reader-action@v2 with: version: ${{ steps.build.outputs.version }} - name: Create GitHub release and upload the distribution id: create-release uses: softprops/action-gh-release@v2 with: name: ${{ steps.build.outputs.version }} body: ${{ steps.read-changelog.outputs.changes }} draft: ${{ env.ACTIONS_STEP_DEBUG == 'true' }} prerelease: ${{ steps.prerelease_check.outputs.prerelease }} files: ${{ steps.build.outputs.asset_path }} - name: Upload the distribution to PyPI if: ${{ env.B2_PYPI_PASSWORD != '' && steps.prerelease_check.outputs.prerelease == 'false' }} uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.B2_PYPI_PASSWORD }} b2-sdk-python-2.8.0/.github/workflows/ci.yml000066400000000000000000000124301474454370000206450ustar00rootroot00000000000000name: Continuous Integration on: push: branches: [master] pull_request: branches: [master] env: PYTHON_DEFAULT_VERSION: "3.12" jobs: lint: timeout-minutes: 30 runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: codespell-project/actions-codespell@2391250ab05295bddd51e36a8c6295edb6343b0e with: ignore_words_list: datas re-use - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" - name: Install dependencies run: python -m pip install --upgrade nox pdm - name: Run linters run: nox -vs lint - name: Validate new changelog entries if: (contains(github.event.pull_request.labels.*.name, '-changelog') == false) && (github.event.pull_request.base.ref != '') run: if [ -z "$(git diff --diff-filter=A --name-only origin/${{ github.event.pull_request.base.ref }} changelog.d)" ]; then echo no changelog item added; exit 1; fi - name: Changelog validation run: nox -vs towncrier_check build: timeout-minutes: 30 needs: lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" - name: Install dependencies run: python -m pip install --upgrade nox pdm - name: Build the distribution run: nox -vs build cleanup_buckets: timeout-minutes: 30 needs: lint env: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" - name: Install dependencies if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead run: python -m pip install --upgrade nox pdm - name: Find and remove old buckets if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead run: nox -vs cleanup_old_buckets test: timeout-minutes: 90 needs: cleanup_buckets env: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} NOX_EXTRAS: ${{ matrix.extras }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "pypy3.9", "pypy3.10"] extras: [ "" ] exclude: - os: "macos-latest" python-version: "pypy3.10" - os: "windows-latest" python-version: "pypy3.10" # Workaround for https://github.com/actions/setup-python/issues/696 - os: "macos-latest" python-version: 3.8 - os: "macos-latest" python-version: 3.9 include: - python-version: "3.12" extras: "full" os: "ubuntu-latest" # Workaround for https://github.com/actions/setup-python/issues/696 - os: "macos-13" python-version: 3.8 - os: "macos-13" python-version: 3.9 steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: "pip" - name: Install dependencies run: python -m pip install --upgrade nox pdm - name: Run unit tests run: nox -vs unit -- -v - name: Run integration tests if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} run: nox -vs integration -- --dont-cleanup-old-buckets -v doc: timeout-minutes: 30 needs: build runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} cache: "pip" - name: Install dependencies env: DEBIAN_FRONTEND: noninteractive run: | sudo apt-get update -y sudo apt-get install -y graphviz plantuml python -m pip install --upgrade nox pdm - name: Build the docs run: nox --non-interactive -vs doc b2-sdk-python-2.8.0/.gitignore000066400000000000000000000002261474454370000161220ustar00rootroot00000000000000*.pyc .codacy-coverage/ .coverage .eggs/ .idea .nox/ .python-version b2sdk.egg-info build coverage.xml dist venv .venv .vscode .pdm-build/ .pdm-pythonb2-sdk-python-2.8.0/.readthedocs.yml000066400000000000000000000013541474454370000172230ustar00rootroot00000000000000# .readthedocs.yml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 build: os: ubuntu-22.04 tools: python: "3.12" apt_packages: - graphviz jobs: post_create_environment: - pip install pdm - pdm export --format requirements --group doc --output requirements-doc.txt # Build documentation in the docs/ directory with Sphinx sphinx: configuration: doc/source/conf.py # Optionally build your docs in additional formats such as PDF and ePub formats: all # Optionally set the version of Python and requirements required to build your docs python: install: - requirements: requirements-doc.txt - method: pip path: . b2-sdk-python-2.8.0/CHANGELOG.md000066400000000000000000000750531474454370000157550ustar00rootroot00000000000000# Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the changes for the upcoming release can be found in [changelog.d](changelog.d). ## [2.8.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.8.0) - 2025-01-23 ### Changed - Migrate to B2 Native API v3. ### Fixed - Fix continuation for started large files with no fully finished parts. - Perform re-authentication for empty 401 responses returned for `HEAD` requests. ### Infrastructure - Remove yapf in favor of ruff. ## [2.7.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.7.0) - 2024-12-12 ### Changed - Make Event Notifications generally available. ([#518](https://github.com/Backblaze/b2-sdk-python/issues/518)) - Switch a pytest hook from path to collection_path. ### Fixed - Add upload token reset after upload timeout. - Fix file/directory permission handling for Windows during the B2 sync. ### Infrastructure - Fix event notification tests when introducing new keys in API outputs. ## [2.6.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.6.0) - 2024-10-28 ### Removed - Remove Python 3.7 support in new releases. Under Python 3.7 `pip` will keep resolving the latest version of the package that supports active interpreter. Python 3.8 is now the minimum supported version, [until it reaches EOL in October 2024](https://devguide.python.org/versions/). We encourage use of the latest stable Python release. ### Fixed - Fixed datetime.utcnow() deprecation warnings under Python 3.12. ### Added - Declare official support for Python 3.13 in b2sdk. Test b2sdk against Python 3.13 in CI. ### Infrastructure - Upgraded to pytest 8 (#484). ## [2.5.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.5.1) - 2024-08-15 ### Fixed - Fix LocalFolder.all_files(..) erroring out if one of the non-excluded directories is not readable by the user running the scan. Warning is added to ProgressReport instead as other file access errors are. ## [2.5.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.5.0) - 2024-07-30 ### Fixed - Fix TruncatedOutput errors when downloading files over congested network (fixes [B2_Command_Line_Tool#554](https://github.com/Backblaze/B2_Command_Line_Tool/issues/554)). - Ensure `FileSimulator.as_download_headers` returns `dict[str, str]` mapping. ### Added - Add `unhide_file` method to Bucket class. ### Doc - Improve `download_file_from_url` methods type hints. ### Infrastructure - Limit max CI (Github Actions) duration to 90 minutes. ## [2.4.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.4.1) - 2024-06-19 ### Fixed - Fix `LocalFolder` regression (introduced in 2.4.0) which caused `LocalFolder` to not list files by path lexicographical order. This is also a fix for `synchronizer` re-uploading files on every run in some cases. ([#502](https://github.com/Backblaze/b2-sdk-python/issues/502)) ## [2.4.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.4.0) - 2024-06-17 ### Changed - In `b2sdk.v3` the `B2Api` will always create `cache` from `AccountInfo` object, unless `cache` is provided explicitly. The current stable `b2sdk.v2` remains unchanged, i.e. `DummyCache` is created by default if `account_info` was provided, but not `cache`. Documentation for `b2sdk.v2` was updated with the new recommended usage, e.g. `B2Api(info, cache=AuthInfoCache(info))`, to achieve the same behavior as `b2sdk.v3`. ([#497](https://github.com/Backblaze/b2-sdk-python/issues/497)) ### Fixed - Move scan filters before a read on filesystem access attempt. This will prevent unnecessary warnings and IO operations on paths that are not relevant to the operation. ([#456](https://github.com/Backblaze/b2-sdk-python/issues/456)) - Fix bucket caching erroring out when using `StubAccountInfo`. ### Added - Add `annotated_types` dependency for type annotations that include basic value validation. - Add `daysFromStartingToCancelingUnfinishedLargeFiles` option to `lifecycle_rules` type annotation. - Add non-retryable `NoPaymentHistory` exception. API returns this exception when action (e.g. bucket creation or replication rules) is not allowed due to lack of payment history. ## [2.3.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.3.0) - 2024-05-15 ### Added - Add `folder_to_list_can_be_a_file` parameter to `b2sdk.v2.Bucket.ls`, that if set to `True` will allow listing a file versions if path is an exact match. This parameter won't be included in `b2sdk.v3.Bucket.ls` and unless supplied `path` ends with `/`, the possibility of path pointing to file will be considered first. ## [2.2.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.2.1) - 2024-05-09 ### Fixed - Fix `__str__` of `b2sdk.v2.BucketIdNotFound` to return full error message and not just missing bucket ID value. ## [2.2.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.2.0) - 2024-05-08 ### Added - Add `has_errors_or_warnings` method to `ProgressReport` class. ### Fixed - Ensure `b2sdk.v2.b2http` emits `b2sdk.v2.BucketIdNotFound` exception instead of `b2sdk._v3.BucketIdNotFound`. ([#437](https://github.com/Backblaze/b2-sdk-python/issues/437)) - Ensure `unprintable_to_hex` and `unprintable_to_hex` return empty string (instead of `None`) if empty string was supplied as argument. - Skip files with invalid filenames when scanning directories (for `sync`, ...) instead of raising an exception. ## [2.1.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.1.0) - 2024-04-15 ### Changed - Use ParallelDownloader for small files instead of SimpleDownloader to avoid blocking on I/O. ### Fixed - Fix `decode_content=True` causing an error when downloading tiny and large files. - Prevent errors due to the use of "seekable" download strategies for seekable, but not readable files. ### Added - Add set&get Event Notification rules methods to Bucket API as part of Event Notifications feature Private Preview. See https://www.backblaze.com/blog/announcing-event-notifications/ for details. ## [2.0.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.0.0) - 2024-04-02 ### Removed - Remove `tqdm` dependency. Now `tqdm` has to be explicitly installed to use `TqdmProgressListener` class. - Remove `[doc]` extras dependency group - moved to dev dependencies. - Remove unnecessary `packaging` package dependency. It's functionality was never explicitly exposed. ### Changed - Move non-apiver packages (e.g. packages other than `b2sdk.v1`, `b2sdk.v2`, ...) to `b2sdk._internal` to further discourage use of non-public internals. If you accidentally used non-public internals, most likely only thing you will need to do, is import from `b2sdk.v2` instead of `b2sdk`. - Move logging setup and `UrllibWarningFilter` class from `b2sdk.__init__.py` to `b2sdk._v3` (and thus `b2sdk.v2` & `b2sdk.v1`). This will allow us to remove/change it in new apiver releases without the need to change the major semver version. ### Added - Add `SqliteAccountInfo.get_user_account_info_path` to public API. ### Infrastructure - Update to [GitHub Actions using Node 20](https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/). ## [1.33.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.33.0) - 2024-03-15 ### Fixed - Escape control characters whenever printing object and bucket names to improve security. - Remove unused `setuptools` from default dependency list. ### Added - Added control character escaping methods. ## [1.32.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.32.0) - 2024-02-26 ### Added - Add `set_thread_pool_size`, `get_thread_pool_size` to *Manger classes. ### Infrastructure - Fix schema graph rendering in readthedocs documentation. ## [1.31.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.31.0) - 2024-02-19 ### Fixed - Remove obsolete test scripts from b2sdk package: `test_upload_url_concurrency`, `b2sdk.b2http:test_http`. ([#471](https://github.com/Backblaze/b2-sdk-python/issues/471)) ### Added - Allow for `min_part_size` that is greater than default `recommended_part_size` value, without having to explicitly set `recommended_part_size` value. - Add `GET` method support to `B2Http`. - Add `JSON` type annotation and fix type hints in `B2Http` methods. - Add more type hints to API methods. ## [1.30.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.30.1) - 2024-02-02 ### Fixed - Fix package author metadata. ## [1.30.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.30.0) - 2024-02-02 ### Fixed - Fix escape sequence warnings present in python 3.12. ([#458](https://github.com/Backblaze/b2-sdk-python/issues/458)) - Handle json encoded, invalid B2 error responses, preventing exceptions such as `invalid literal for int() with base 10: 'service_unavailable'`. ### Added - Add support for filters to `Bucket.ls()`. ### Infrastructure - Package the library using [pdm](https://pdm-project.org), use locked dependencies in CI. - Update `ruff` linter and apply it to all files. ## [1.29.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.29.1) - 2024-01-23 ### Fixed - Handle non-json encoded B2 error responses, i.e. retry on 502 and 504 errors. ### Doc - Add missing import in Synchronizer docs example. ## [1.29.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.29.0) - 2023-12-13 ### Changed - Change v3.B2Api.authorize_account signature to make `realm` optional and `"production"` by default. ### Added - Progress listener instances can now change their descriptions during run. This allows for e.g.: changing description after file headers are downloaded but before the content is fetched. ### Infrastructure - Add `-v` to pytest in CI. - Run windows pypy3.9 tests on nightly builds. ## [1.28.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.28.0) - 2023-12-06 ### Changed - On XDG compatible OSes (Linux, BSD), the profile file is now created in `$XDG_CONFIG_HOME` (with a fallback to `~/.config/` in absence of given env. variable). - Replace blank `assert` with exception when size values for parts upload are misaligned. ### Fixed - Streaming from empty stream no longer ends with "Empty emerge parts iterator" error. ### Infrastructure - Changelog entries are now validated as a part of CI pipeline. - Disable dependabot requests for updates unrelated to security issues. - Fixed tests failing because of changes made to `locale.normalize` in Python 3.12. ## [1.27.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.27.0) - 2023-11-26 ### Changed - Add dependency on `setuptools` and `packaging` as they are not shipped by cpython 3.12 and are used in production code. ### Fixed - Fix closing of passed progress listeners in `Bucket.upload` and `Bucket.copy` ## [1.26.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.26.0) - 2023-11-20 ### Added - Add `expires`, `content_disposition`, `content_encoding`, `content_language` arguments to various `Bucket` methods ([#357](https://github.com/Backblaze/b2-sdk-python/issues/357)) ### Infrastructure - Towncrier changelog generation - to avoid conflicts when simultaneously working on PRs - Fix towncrier generated changelog to work with mindsers/changelog-reader-action ## [1.25.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.25.0) - 2023-11-15 ### Added - Add `*_PART_SIZE`, `BUCKET_NAME_*`, `STDOUT_FILEPATH` constants - Add `points_to_fifo`, `points_to_stdout` functions ### Changed - Mark `TempDir` as deprecated in favor of `tempfile.TemporaryDirectory` ### Fixed - Fix downloading to a non-seekable file, such as /dev/stdout - Fix ScanPoliciesManager support for compiled regexes ### Infrastructure - Fix readthedocs build by updating to v2 configuration schema - Fix spellcheck erroring out on LICENSE file - Fix snyk reporting vulnerability due to tornado package use in docs generation - Deduplicate test_base files in test suite - Refactor integration tests for better pytest compatibility & eager bucket cleanup ## [1.24.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.24.1) - 2023-09-27 ### Fixed - Fix missing key ID for large file encrypted with SSE-C - Fix concatenating error message when message is None ## [1.24.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.24.0) - 2023-08-31 ### Added - 'bypass_governance' flag to delete_file_version ## [1.23.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.23.0) - 2023-08-10 ### Added - Add `get_file_info_by_name` to the B2Api class ### Fixed - Require `typing_extensions` on Python 3.11 (already required on earlier versions) for better compatibility with pydantic v2 - Fix `RawSimulator` handling of `cache_control` parameter during tests. ## [1.22.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.22.1) - 2023-07-24 ### Fixed - Fix regression in dir exclusion patterns introduced in 1.22.0 ## [1.22.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.22.0) - 2023-07-21 ### Added - Declare official support of Python 3.12 - Improved `lifecycle_rules` argument type annotations ### Deprecated - Deprecate `file_infos` argument. Use `file_info` instead. Old argument name won't be supported in v3. ### Changed - `version_utils` decorators now ignore `current_version` parameter to better fit `apiver` needs ### Fixed - Circular symlinks no longer cause infinite loops when syncing a folder - Fix crash on upload retry with unbound data source ### Infrastructure - Remove unsupported PyPy versions (3.7, 3.8) from tests matrix and add PyPy 3.9 & 3.10 instead - Replaced `pyflakes` with `ruff` for linting - Refactored logic for resuming large file uploads to unify code paths, correct inconsistencies, and enhance configurability (#381) - Automatically set copyright date when generating the docs - Use modern type hints in documentation (achieved through combination of PEP 563 & 585 and `sphinx-autodoc-typehints`) ## [1.21.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.21.0) - 2023-04-17 ### Added - Add support for custom upload timestamp - Add support for cache control header while uploading ### Infrastructure - Remove dependency from `arrow` - Build Python wheels for distribution ## [1.20.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.20.0) - 2023-03-23 ### Added - Add `use_cache` parameter to `B2Api.list_buckets` ### Changed - Connection timeout is now being set explicitly ### Fixed - Small files downloaded twice ### Infrastructure - Disable changelog verification for dependabot PRs ## [1.19.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.19.0) - 2023-01-24 ### Added - Authorizing a key for a single bucket ensures that this bucket is cached - `Bucket.ls` operation supports wildcard matching strings - Documentation for `AbstractUploadSource` and its children - `InvalidJsonResponse` when the received error is not a proper JSON document - Raising `PotentialS3EndpointPassedAsRealm` when a specific misconfiguration is suspected - Add `large_file_sha1` support - Add support for incremental upload and sync - Ability to stream data from an unbound source to B2 (for example stdin) ### Fixed - Removed information about replication being in closed beta - Don't throw raw `OSError` exceptions when using `DownloadedFile.save_to` to a path that doesn't exist, is a directory or the user doesn't have permissions to write to ### Infrastructure - Additional tests for listing files/versions - Ensured that changelog validation only happens on pull requests - Upgraded GitHub actions checkout to v3, python-setup to v4 - Additional tests for `IncrementalHexDigester` ## [1.18.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.18.0) - 2022-09-20 ### Added - Logging performance summary of parallel download threads - Add `max_download_streams_per_file` parameter to B2Api class and underlying structures - Add `is_file_lock_enabled` parameter to `Bucket.update()` and related methods ### Fixed - Replace `ReplicationScanResult.source_has_sse_c_enabled` with `source_encryption_mode` - Fix `B2Api.get_key()` and `RawSimulator.delete_key()` - Fix calling `CopySizeTooBig` exception ### Infrastructure - Fix nox's deprecated `session.install()` calls - Re-enable changelog validation in CI - StatsCollector contains context managers for gathering performance statistics ## [1.17.3](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.3) - 2022-07-15 ### Fixed - Fix `FileVersion._get_upload_headers` when encryption key is `None` ### Infrastructure - Fix download integration tests on non-production environments - Add `B2_DEBUG_HTTP` env variable to enable network-level test debugging - Disable changelog validation temporarily ## [1.17.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.2) - 2022-06-24 ### Fixed - Fix a race in progress reporter - Fix import of replication ## [1.17.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.1) - 2022-06-23 [YANKED] ### Fixed - Fix importing scan module ## [1.17.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.0) - 2022-06-23 [YANKED] As in version 1.16.0, the replication API may still be unstable, however no backward-incompatible changes are planned at this point. ### Added - Add `included_sources` module for keeping track of included modified third-party libraries - Add `include_existing_files` parameter to `ReplicationSetupHelper` - Add `get_b2sdk_doc_urls` function for extraction of external documentation URLs during runtime ### Changed - Downloading compressed files with `Content-Encoding` header set no longer causes them to be decompressed on the fly - it's an option - Change the per part retry limit from 5 to 20 for data transfer operations. Please note that the retry system is not considered to be a part of the public interface and is subject to be adjusted - Do not wait more than 64 seconds between retry attempts (unless server asks for it) - On longer failures wait an additional (random, up to 1s) amount of time to prevent client synchronization - Flatten `ReplicationConfiguration` interface - Reorder actions of `ReplicationSetupHelper` to avoid zombie rules ### Fixed - Fix: downloading compressed files and decompressing them on the fly now does not cause a TruncatedOutput error - Fix `AccountInfo.is_master_key()` - Fix docstring of `SqliteAccountInfo` - Fix lifecycle rule type in the docs ### Infrastructure - Add 3.11.0-beta.1 to CI - Change Sphinx major version from 5 to 6 - Extract folder/bucket scanning into a new `scan` module - Enable pip cache in CI ## [1.16.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.16.0) - 2022-04-27 This release contains a preview of replication support. It allows for basic usage of B2 replication feature (currently in closed beta). As the interface of the sdk (and the server api) may change, the replication support shall be considered PRIVATE interface and should be used with caution. Please consult the documentation on how to safely use the private api interface. Expect substantial amount of work on sdk interface: - The interface of `ReplicationConfiguration` WILL change - The interface of `FileVersion.replication_status` MIGHT change - The interface of `FileVersionDownload` MIGHT change ### Added - Add basic replication support to `Bucket` and `FileVersion` - Add `is_master_key()` method to `AbstractAccountInfo` - Add `readBucketReplications` and `writeBucketReplications` to `ALL_CAPABILITIES` - Add log tracing of `interpret_b2_error` - Add `ReplicationSetupHelper` ### Fixed - Fix license test on Windows - Fix cryptic errors when running integration tests with a non-full key ## [1.15.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.15.0) - 2022-04-12 ### Changed - Don't run coverage in pypy in CI - Introduce a common thread worker pool for all downloads - Increase http timeout to 20min (for copy using 5GB parts) - Remove inheritance from object (leftover from python2) - Run unit tests on all CPUs ### Added - Add pypy-3.8 to test matrix - Add support for unverified checksum upload mode - Add dedicated exception for unverified email - Add a parameter to customize `sync_policy_manager` - Add parameters to set the min/max part size for large file upload/copy methods - Add CopySourceTooBig exception - Add an option to set a custom file version class to `FileVersionFactory` - Add an option for B2Api to turn off hash checking for downloaded files - Add an option for B2Api to set write buffer size for `DownloadedFile.save_to` method - Add support for multiple profile files for SqliteAccountInfo ### Fixed - Fix copying objects larger than 1TB - Fix uploading objects larger than 1TB - Fix downloading files with unverified checksum - Fix decoding in filename and file info of `DownloadVersion` - Fix an off-by-one bug and other bugs in the Simulator copy functionality ### Removed - Drop support for Python 3.5 and Python 3.6 ## [1.14.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.14.1) - 2022-02-23 ### Security - Fix setting permissions for local sqlite database (thanks to Jan Schejbal for responsible disclosure!) ## [1.14.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.14.0) - 2021-12-23 ### Fixed - Relax constraint on arrow to allow for versions >= 1.0.2 ## [1.13.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.13.0) - 2021-10-24 ### Added - Add support for Python 3.10 ### Changed - Update a list with all capabilities ### Fixed - Fix pypy selector in CI ## [1.12.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.12.0) - 2021-08-06 ### Changed - The `importlib-metadata` requirement is less strictly bound now (just >=3.3.0 for python > 3.5). - `B2Api` `update_file_legal_hold` and `update_file_retention_setting` now return the set values ### Added - `BucketIdNotFound` thrown based on B2 cloud response - `_clone` method to `FileVersion` and `DownloadVersion` - `delete`, `update_legal_hold`, `update_retention` and `download` methods added to `FileVersion` ### Fixed - FileSimulator returns special file info headers properly ### Removed - One unused import. ## [1.11.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.11.0) - 2021-06-24 ### Changed - apiver `v2` interface released. `from b2sdk.v2 import ...` is now the recommended import, but `from b2sdk.v1 import ...` works as before ## [1.10.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.10.0) - 2021-06-23 ### Added - `get_fresh_state` method added to `FileVersion` and `Bucket` ### Changed - `download_file_*` methods refactored to allow for inspecting DownloadVersion before downloading the whole file - `B2Api.get_file_info` returns a `FileVersion` object in v2 - `B2RawApi` renamed to `B2RawHTTPApi` - `B2HTTP` tests are now common - `B2HttpApiConfig` class introduced to provide parameters like `user_agent_append` to `B2Api` without using internal classes in v2 - `Bucket.update` returns a `Bucket` object in v2 - `Bucket.ls` argument `show_versions` renamed to `latest_only` in v2 - `B2Api` application key methods refactored to operate with dataclasses instead of dicts in v2 - `B2Api.list_keys` is a generator lazily fetching all keys in v2 - `account_id` and `bucket_id` added to FileVersion ### Fixed - Fix EncryptionSetting.from_response_headers - Fix FileVersion.size and FileVersion.mod_time_millis type ambiguity - Old buckets (from past tests) are cleaned up before running integration tests in a single thread ### Removed - Remove deprecated `SyncReport` methods ## [1.9.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.9.0) - 2021-06-07 ### Added - `ScanPoliciesManager` is able to filter b2 files by upload timestamp ### Changed - `Synchronizer.make_file_sync_actions` and `Synchronizer.make_folder_sync_actions` were made private in v2 interface - Refactored `sync.file.*File` and `sync.file.*FileVersion` to `sync.path.*SyncPath` in v2 - Refactored `FileVersionInfo` to `FileVersion` in v2 - `ScanPoliciesManager` exclusion interface changed in v2 - `B2Api` unittests for v0, v1 and v2 are now common - `B2Api.cancel_large_file` returns a `FileIdAndName` object instead of a `FileVersion` object in v2 - `FileVersion` has a mandatory `api` parameter in v2 - `B2Folder` holds a handle to B2Api - `Bucket` unit tests for v1 and v2 are now common ### Fixed - Fix call to incorrect internal api in `B2Api.get_download_url_for_file_name` ## [1.8.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.8.0) - 2021-05-21 ### Added - Add `get_bucket_name_or_none_from_bucket_id` to `AccountInfo` and `Cache` - Add possibility to change realm during integration tests - Add support for "file locks": file retention, legal hold and default bucket retention ### Fixed - Cleanup sync errors related to directories - Use proper error handling in `ScanPoliciesManager` - Application key restriction message reverted to previous form - Added missing apiver wrappers for FileVersionInfo - Fix crash when Content-Range header is missing - Pin dependency versions appropriately ### Changed - `b2sdk.v1.sync` refactored to reflect `b2sdk.sync` structure - Make `B2Api.get_bucket_by_id` return populated bucket objects in v2 - Add proper support of `recommended_part_size` and `absolute_minimum_part_size` in `AccountInfo` - Refactored `minimum_part_size` to `recommended_part_size` (the value used stays the same) - Encryption settings, types and providers are now part of the public API ### Removed - Remove `Bucket.copy_file` and `Bucket.start_large_file` - Remove `FileVersionInfo.format_ls_entry` and `FileVersionInfo.format_folder_ls_entry` ## [1.7.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.7.0) - 2021-04-22 ### Added - Add `__slots__` and `__eq__` to `FileVersionInfo` for memory usage optimization and ease of testing - Add support for SSE-C server-side encryption mode - Add support for `XDG_CONFIG_HOME` for determining the location of `SqliteAccountInfo` db file ### Changed - `BasicSyncEncryptionSettingsProvider` supports different settings sets for reading and writing - Refactored AccountInfo tests to a single file using pytest ### Fixed - Fix clearing cache during `authorize_account` - Fix `ChainedStream` (needed in `Bucket.create_file` etc.) - Make tqdm-based progress reporters less jumpy and easier to read - Fix emerger examples in docs ## [1.6.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.6.0) - 2021-04-08 ### Added - Fetch S3-compatible API URL from `authorize_account` ### Fixed - Exclude packages inside the test package when installing - Fix for server response change regarding SSE ## [1.5.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.5.0) - 2021-03-25 ### Added - Add `dependabot.yml` - Add support for SSE-B2 server-side encryption mode ### Changed - Add upper version limit for the requirements ### Fixed - Pin `setuptools-scm<6.0` as `>=6.0` doesn't support Python 3.5 ## [1.4.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.4.0) - 2021-03-03 ### Changed - Add an ability to provide `bucket_id` filter parameter for `list_buckets` - Add `is_same_key` method to `AccountInfo` - Add upper version limit for arrow dependency, because of a breaking change ### Fixed - Fix docs autogen ## [1.3.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.3.0) - 2021-01-13 ### Added - Add custom exception for `403 transaction_cap_exceeded` - Add `get_file_info_by_id` and `get_file_info_by_name` to `Bucket` - `FileNotPresent` and `NonExistentBucket` now subclass new exceptions `FileOrBucketNotFound` and `ResourceNotFound` ### Changed - Fix missing import in the synchronization example - Use `setuptools-scm` for versioning - Clean up CI steps ## [1.2.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.2.0) - 2020-11-03 ### Added - Add support for Python 3.9 - Support for bucket to bucket sync - Add a possibility to append a string to the User-Agent in `B2Http` ### Changed - Change default fetch count for `ls` to 10000 ### Removed - Drop Python 2 and Python 3.4 support :tada: - Remove `--prefix` from `ls` (it didn't really work, use `folderName` argument) ### Fixed - Allow to set an empty bucket info during the update - Fix docs generation in CI ## [1.1.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.1.4) - 2020-07-15 ### Added - Allow specifying custom realm in B2Session.authorize_account ## [1.1.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.1.2) - 2020-07-06 ### Fixed - Fix upload part for file range on Python 2.7 ## [1.1.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.1.0) - 2020-06-24 ### Added - Add `list_file_versions` method to buckets. - Add server-side copy support for large files - Add ability to synthesize objects from local and remote sources - Add AuthInfoCache, InMemoryCache and AbstractCache to public interface - Add ability to filter in ScanPoliciesManager based on modification time - Add ScanPoliciesManager and SyncReport to public interface - Add md5 checksum to FileVersionInfo - Add more keys to dicts returned by as_dict() methods ### Changed - Make sync treat hidden files as deleted - Ignore urllib3 "connection pool is full" warning ### Removed - Remove arrow warnings caused by https://github.com/crsmithdev/arrow/issues/612 ### Fixed - Fix handling of modification time of files ## [1.0.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.0.2) - 2019-10-15 ### Changed - Remove upper version limit for arrow dependency ## [1.0.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.0.0) - 2019-10-03 ### Fixed - Minor bug fix. ## [1.0.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.0.0)-rc1 - 2019-07-09 ### Deprecated - Deprecate some transitional method names to v0 in preparation for v1.0.0. ## [0.1.10](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.10) - 2019-07-09 ### Removed - Remove a parameter (which did nothing, really) from `b2sdk.v1.Bucket.copy_file` signature ## [0.1.8](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.8) - 2019-06-28 ### Added - Add support for b2_copy_file - Add support for `prefix` parameter on ls-like calls ## [0.1.6](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.6) - 2019-04-24 ### Changed - Rename account ID for authentication to application key ID. Account ID is still backwards compatible, only the terminology has changed. ### Fixed - Fix transferer crashing on empty file download attempt ## [0.1.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.4) - 2019-04-04 ### Added Initial official release of SDK as a separate package (until now it was a part of B2 CLI) b2-sdk-python-2.8.0/CONTRIBUTING.md000066400000000000000000000114751474454370000163730ustar00rootroot00000000000000# Contributing to B2 Python SDK We encourage outside contributors to perform changes on our codebase. Many such changes have been merged already. In order to make it easier to contribute, core developers of this project: * provide guidance (through the issue reporting system) * provide tool assisted code review (through the Pull Request system) * maintain a set of unit tests * maintain a set of integration tests (run with a production cloud) * maintain development automation tools using [nox](https://github.com/theacodes/nox) that can easily: * format the code using [ruff](https://github.com/astral-sh/ruff) * runs linters to find subtle/potential issues with maintainability * run the test suite on multiple Python versions using [pytest](https://github.com/pytest-dev/pytest) * maintain Continuous Integration (by using GitHub Actions) that: * runs all sorts of linters * checks if the Python distribution can be built * runs all tests on a matrix of 6 versions of Python (including pypy) and 3 operating systems (Linux, Mac OS X and Windows) * checks if the documentation can be built properly * maintain other Continuous Integration tools (coverage tracker) ## Versioning This package's versions adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and the versions are established by reading git tags, i.e. no code or manifest file changes are required when working on PRs. ## Changelog Each PR needs to have at least one changelog (aka news) item added. This is done by creating files in `changelog.d`. `towncrier` is used for compiling these files into [CHANGELOG.md](CHANGELOG.md). There are several types of changes (news): 1. fixed 2. changed 3. added 4. deprecated 5. removed 6. infrastructure 7. doc The `changelog.d` file name convention is: 1. If the PR closes a github issue: `{issue_number}.{type}.md` e.g. `157.fixed.md`. Note that the change description still has to be complete, linking an issue is just there for convenience, a change like `fixed #157` will not be accepted. 2. If the PR is not related to a github issue: `+{unique_string}.{type}.md` e.g. `+foobar.fixed.md`. These files can either be created manually, or using `towncrier` e.g. towncrier create -c 'write your description here' 157.fixed.md `towncrier create` also takes care of duplicates automatically (if there is more than 1 news fragment of one type for a given github issue). ## Developer Info You'll need to have [nox](https://github.com/theacodes/nox) and [pdm](https://pdm-project.org/) installed: * `pip install nox pdm` With `nox`, you can run different sessions (default are `lint` and `test`): * `format` -> Format the code. * `lint` -> Run linters. * `test` (`test-3.7`, `test-3.8`, `test-3.9`, `test-3.10`) -> Run test suite. * `cover` -> Perform coverage analysis. * `build` -> Build the distribution. * `doc` -> Build the documentation. * `doc_cover` -> Perform coverage analysis for the documentation. For example: $ nox -s format nox > Running session format nox > Creating virtual environment (virtualenv) using python3.10 in .nox/format ... $ nox -s format nox > Running session format nox > Re-using existing virtual environment at .nox/format. ... $ nox --no-venv -s format nox > Running session format ... Sessions `test` ,`unit`, and `integration` can run on many Python versions, 3.7-3.10 by default. Sessions other than `test` use the last given Python version, 3.10 by default. You can change it: export NOX_PYTHONS=3.7,3.8 With the above setting, session `test` will run on Python 3.7 and 3.8, and all other sessions on Python 3.8. Given Python interpreters should be installed in the operating system or via [pyenv](https://github.com/pyenv/pyenv). ## Managing dependencies We use [pdm](https://pdm-project.org/) for managing dependencies and developing locally. If you want to change any of the project requirements (or requirement bounds) in `pyproject.toml`, make sure that `pdm.lock` file reflects those changes by using `pdm add`, `pdm update` or other commands - see [documentation](https://pdm-project.org/latest/). You can verify that lock file is up to date by running the linter. ## Linting To run all available linters: nox -s lint ## Testing To run all tests on every available Python version: nox -s test To run all tests on a specific version: nox -s test-3.10 To run just unit tests: nox -s unit-3.10 To run just integration tests: export B2_TEST_APPLICATION_KEY=your_app_key export B2_TEST_APPLICATION_KEY_ID=your_app_key_id nox -s integration-3.10 To run tests by keyword expressions: nox -s unit-3.10 -- -k keyword ## Documentation To build the documentation and watch for changes (including the source code): nox -s doc To just build the documentation: nox --non-interactive -s doc b2-sdk-python-2.8.0/LICENSE000066400000000000000000000272731474454370000151520ustar00rootroot00000000000000Backblaze wants developers and organization to copy and re-use our code examples, so we make the samples available by several different licenses. One option is the MIT license (below). Other options are available here: https://www.backblaze.com/using_b2_code.html The MIT License (MIT) Copyright (c) 2015 Backblaze Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Backblaze has also utilized other open source code (psf/requests) licensed under the Apache License, Version 2.0, January 2004. That Apache License requires that we also provide you will a copy of the Apache License (below, and accessible at https://www.apache.org/licenses/LICENSE-2.0). Note that Backblaze does not license its own code to you under the Apache License, but simply provides you a copy of such Apache License to comply with its own license obligations. Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. b2-sdk-python-2.8.0/README.md000066400000000000000000000042531474454370000154150ustar00rootroot00000000000000# B2 Python SDK  [![Continuous Integration](https://github.com/Backblaze/b2-sdk-python/workflows/Continuous%20Integration/badge.svg)](https://github.com/Backblaze/b2-sdk-python/actions?query=workflow%3A%22Continuous+Integration%22) [![License](https://img.shields.io/pypi/l/b2sdk.svg?label=License)](https://pypi.python.org/pypi/b2) [![python versions](https://img.shields.io/pypi/pyversions/b2sdk.svg?label=python%20versions)](https://pypi.python.org/pypi/b2sdk) [![PyPI version](https://img.shields.io/pypi/v/b2sdk.svg?label=PyPI%20version)](https://pypi.python.org/pypi/b2sdk) [![Docs](https://readthedocs.org/projects/b2-sdk-python/badge/?version=master)](https://b2-sdk-python.readthedocs.io/en/master/) This repository contains a client library and a few handy utilities for easy access to all of the capabilities of B2 Cloud Storage. [B2 command-line tool](https://github.com/Backblaze/B2_Command_Line_Tool) is an example of how it can be used to provide command-line access to the B2 service, but there are many possible applications (including FUSE filesystems, storage backend drivers for backup applications etc). # Documentation The latest documentation is available on [Read the Docs](https://b2-sdk-python.readthedocs.io). # Installation The sdk can be installed with: pip install b2sdk # Version policy b2sdk follows [Semantic Versioning](https://semver.org/) policy, so in essence the version number is MAJOR.MINOR.PATCH (for example 1.2.3) and: - we increase MAJOR version when we make incompatible API changes - we increase MINOR version when we add functionality in a backwards-compatible manner, and - we increase PATCH version when we make backwards-compatible bug fixes (unless someone relies on the undocumented behavior of a fixed bug) Therefore when setting up b2sdk as a dependency, please make sure to match the version appropriately, for example you could put this in your `requirements.txt` to make sure your code is compatible with the `b2sdk` version your user will get from pypi: ``` b2sdk>=2,<3 ``` # Release History Please refer to the [changelog](CHANGELOG.md). # Developer Info Please see our [contributing guidelines](CONTRIBUTING.md). b2-sdk-python-2.8.0/README.release.md000066400000000000000000000002121474454370000170230ustar00rootroot00000000000000# Release Process - Run `nox -s make_release_commit -- X.Y.Z` where `X.Y.Z` is the version you're releasing, and follow the instructions b2-sdk-python-2.8.0/b2sdk/000077500000000000000000000000001474454370000151375ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/LICENSE000077700000000000000000000000001474454370000173572../LICENSEustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/__init__.py000066400000000000000000000006471474454370000172570ustar00rootroot00000000000000###################################################################### # # File: b2sdk/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import b2sdk.version # noqa: E402 __version__ = b2sdk.version.VERSION assert __version__ # PEP-0396 b2-sdk-python-2.8.0/b2sdk/__main__.py000066400000000000000000000004771474454370000172410ustar00rootroot00000000000000###################################################################### # # File: b2sdk/__main__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/000077500000000000000000000000001474454370000171125ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/__init__.py000066400000000000000000000006731474454370000212310ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/__init__.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### """ b2sdk._internal package contains internal modules, and should not be used directly. Please use chosen apiver package instead, e.g. b2sdk.v2 """ b2-sdk-python-2.8.0/b2sdk/_internal/account_info/000077500000000000000000000000001474454370000215615ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/account_info/__init__.py000066400000000000000000000006361474454370000236770ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/account_info/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from .in_memory import InMemoryAccountInfo assert InMemoryAccountInfo b2-sdk-python-2.8.0/b2sdk/_internal/account_info/abstract.py000066400000000000000000000311411474454370000237360ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/account_info/abstract.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from abc import abstractmethod from b2sdk._internal.account_info import exception from b2sdk._internal.raw_api import ALL_CAPABILITIES from b2sdk._internal.utils import B2TraceMetaAbstract, limit_trace_arguments class AbstractAccountInfo(metaclass=B2TraceMetaAbstract): """ Abstract class for a holder for all account-related information that needs to be kept between API calls and between invocations of the program. This includes: account ID, application key ID, application key, auth tokens, API URL, download URL, and uploads URLs. This class must be THREAD SAFE because it may be used by multiple threads running in the same Python process. It also needs to be safe against multiple processes running at the same time. """ # The 'allowed' structure to use for old account info that was saved without 'allowed'. DEFAULT_ALLOWED = dict( bucketId=None, bucketName=None, capabilities=ALL_CAPABILITIES, namePrefix=None, ) @classmethod def all_capabilities(cls): """ Return a list of all possible capabilities. :rtype: list """ return ALL_CAPABILITIES @abstractmethod def clear(self): """ Remove all stored information. """ @abstractmethod def list_bucket_names_ids(self) -> list[tuple[str, str]]: """ List buckets in the cache. :return: list of tuples (bucket_name, bucket_id) """ pass @abstractmethod @limit_trace_arguments(only=['self']) def refresh_entire_bucket_name_cache(self, name_id_iterable): """ Remove all previous name-to-id mappings and stores new ones. :param iterable name_id_iterable: an iterable of tuples of the form (name, id) """ @abstractmethod def remove_bucket_name(self, bucket_name): """ Remove one entry from the bucket name cache. :param str bucket_name: a bucket name """ @abstractmethod def save_bucket(self, bucket): """ Remember the ID for the given bucket name. :param b2sdk.v2.Bucket bucket: a Bucket object """ @abstractmethod def get_bucket_id_or_none_from_bucket_name(self, bucket_name): """ Look up the bucket ID for the given bucket name. :param str bucket_name: a bucket name :return bucket ID or None: :rtype: str, None """ @abstractmethod def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> str | None: """ Look up the bucket name for the given bucket id. """ @abstractmethod def clear_bucket_upload_data(self, bucket_id): """ Remove all upload URLs for the given bucket. :param str bucket_id: a bucket ID """ def is_same_key(self, application_key_id, realm): """ Check whether cached application key is the same as the one provided. :param str application_key_id: application key ID :param str realm: authorization realm :rtype: bool """ try: return self.get_application_key_id() == application_key_id and self.get_realm() == realm except exception.MissingAccountData: return False def is_same_account(self, account_id: str, realm: str) -> bool: """ Check whether cached account is the same as the one provided. :param str account_id: account ID :param str realm: authorization realm :rtype: bool """ try: return self.get_account_id() == account_id and self.get_realm() == realm except exception.MissingAccountData: return False def is_master_key(self) -> bool: application_key_id = self.get_application_key_id() account_id = self.get_account_id() new_style_master_key_suffix = '0000000000' if account_id == application_key_id: return True # old style if len(application_key_id) == ( 3 + len(account_id) + len(new_style_master_key_suffix) ): # 3 for cluster id # new style if application_key_id.endswith(account_id + new_style_master_key_suffix): return True return False @abstractmethod def get_account_id(self): """ Return account ID or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_application_key_id(self): """ Return the application key ID used to authenticate. :rtype: str """ @abstractmethod def get_account_auth_token(self): """ Return account_auth_token or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_api_url(self): """ Return api_url or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_application_key(self): """ Return application_key or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_download_url(self): """ Return download_url or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_realm(self): """ Return realm or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @abstractmethod def get_recommended_part_size(self): """ Return the recommended number of bytes in a part of a large file. :return: number of bytes :rtype: int """ @abstractmethod def get_absolute_minimum_part_size(self): """ Return the absolute minimum number of bytes in a part of a large file. :return: number of bytes :rtype: int """ @abstractmethod def get_allowed(self): """ An 'allowed' dict, as returned by ``b2_authorize_account``. Never ``None``; for account info that was saved before 'allowed' existed, returns :attr:`DEFAULT_ALLOWED`. :rtype: dict """ @abstractmethod def get_s3_api_url(self): """ Return s3_api_url or raises :class:`~b2sdk.v2.exception.MissingAccountData` exception. :rtype: str """ @limit_trace_arguments( only=[ 'self', 'api_url', 'download_url', 'recommended_part_size', 'absolute_minimum_part_size', 'realm', 's3_api_url', ] ) def set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id, ): """ Check permission correctness and stores the results of ``b2_authorize_account``. The allowed structure is the one returned by ``b2_authorize_account``, e.g. .. code-block:: python { "absoluteMinimumPartSize": 5000000, "accountId": "YOUR_ACCOUNT_ID", "allowed": { "bucketId": "BUCKET_ID", "bucketName": "BUCKET_NAME", "capabilities": [ "listBuckets", "listFiles", "readFiles", "shareFiles", "writeFiles", "deleteFiles" ], "namePrefix": null }, "apiUrl": "https://apiNNN.backblazeb2.com", "authorizationToken": "4_0022623512fc8f80000000001_0186e431_d18d02_acct_tH7VW03boebOXayIc43-sxptpfA=", "downloadUrl": "https://f002.backblazeb2.com", "recommendedPartSize": 100000000, "s3ApiUrl": "https://s3.us-west-NNN.backblazeb2.com" } For keys with bucket restrictions, the name of the bucket is looked up and stored as well. The console_tool does everything by bucket name, so it's convenient to have the restricted bucket name handy. :param str account_id: user account ID :param str auth_token: user authentication token :param str api_url: an API URL :param str download_url: path download URL :param int recommended_part_size: recommended size of a file part :param int absolute_minimum_part_size: minimum size of a file part :param str application_key: application key :param str realm: a realm to authorize account in :param dict allowed: the structure to use for old account info that was saved without 'allowed' :param str application_key_id: application key ID :param str s3_api_url: S3-compatible API URL .. versionchanged:: 0.1.5 `account_id_or_app_key_id` renamed to `application_key_id` """ if allowed is None: allowed = self.DEFAULT_ALLOWED assert self.allowed_is_valid(allowed) self._set_auth_data( account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id, ) @classmethod def allowed_is_valid(cls, allowed): """ Make sure that all of the required fields are present, and that bucketId is set if bucketName is. If the bucketId is for a bucket that no longer exists, or the capabilities do not allow for listBuckets, then we will not have a bucketName. :param dict allowed: the structure to use for old account info that was saved without 'allowed' :rtype: bool """ return ( ('bucketId' in allowed) and ('bucketName' in allowed) and ((allowed['bucketId'] is not None) or (allowed['bucketName'] is None)) and ('capabilities' in allowed) and ('namePrefix' in allowed) ) @abstractmethod def _set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id, ): """ Actually store the auth data. Can assume that 'allowed' is present and valid. All of the information returned by ``b2_authorize_account`` is saved, because all of it is needed at some point. """ @abstractmethod def take_bucket_upload_url(self, bucket_id): """ Return a pair (upload_url, upload_auth_token) that has been removed from the pool for this bucket, or (None, None) if there are no more left. :param str bucket_id: a bucket ID :rtype: tuple """ @abstractmethod @limit_trace_arguments(only=['self', 'bucket_id']) def put_bucket_upload_url(self, bucket_id, upload_url, upload_auth_token): """ Add an (upload_url, upload_auth_token) pair to the pool available for the bucket. :param str bucket_id: a bucket ID :param str upload_url: an upload URL :param str upload_auth_token: an upload authentication token :rtype: tuple """ @abstractmethod @limit_trace_arguments(only=['self']) def put_large_file_upload_url(self, file_id, upload_url, upload_auth_token): """ Put a large file upload URL into a pool. :param str file_id: a file ID :param str upload_url: an upload URL :param str upload_auth_token: an upload authentication token """ pass @abstractmethod def take_large_file_upload_url(self, file_id): """ Take the chosen large file upload URL from the pool. :param str file_id: a file ID """ pass @abstractmethod def clear_large_file_upload_urls(self, file_id): """ Clear the pool of URLs for a given file ID. :param str file_id: a file ID """ pass b2-sdk-python-2.8.0/b2sdk/_internal/account_info/exception.py000066400000000000000000000025711474454370000241360ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/account_info/exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from abc import ABCMeta from ..exception import B2Error class AccountInfoError(B2Error, metaclass=ABCMeta): """ Base class for all account info errors. """ pass class CorruptAccountInfo(AccountInfoError): """ Raised when an account info file is corrupted. """ def __init__(self, file_name): """ :param file_name: an account info file name :type file_name: str """ super().__init__() self.file_name = file_name def __str__(self): return ( f'Account info file ({self.file_name}) appears corrupted. ' f'Try removing and then re-authorizing the account.' ) class MissingAccountData(AccountInfoError): """ Raised when there is no account info data available. """ def __init__(self, key): """ :param key: a key for getting account data :type key: str """ super().__init__() self.key = key def __str__(self): return f'Missing account data: {self.key}' b2-sdk-python-2.8.0/b2sdk/_internal/account_info/in_memory.py000066400000000000000000000106011474454370000241270ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/account_info/in_memory.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from functools import wraps from .exception import MissingAccountData from .upload_url_pool import UrlPoolAccountInfo def _raise_missing_if_result_is_none(function): """ Raise MissingAccountData if function's result is None. """ @wraps(function) def inner(*args, **kwargs): assert function.__name__.startswith('get_') result = function(*args, **kwargs) if result is None: # assumes that it is a "get_field_name" raise MissingAccountData(function.__name__[4:]) return result return inner class InMemoryAccountInfo(UrlPoolAccountInfo): """ *AccountInfo* which keeps all data in memory. """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._clear_in_memory_account_fields() def clear(self): self._clear_in_memory_account_fields() return super().clear() def _clear_in_memory_account_fields(self): self._account_id = None self._application_key_id = None self._allowed = None self._api_url = None self._application_key = None self._auth_token = None self._buckets = {} self._download_url = None self._recommended_part_size = None self._absolute_minimum_part_size = None self._realm = None self._s3_api_url = None def _set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id, ): self._account_id = account_id self._application_key_id = application_key_id self._auth_token = auth_token self._api_url = api_url self._download_url = download_url self._absolute_minimum_part_size = absolute_minimum_part_size self._recommended_part_size = recommended_part_size self._application_key = application_key self._realm = realm self._s3_api_url = s3_api_url self._allowed = allowed def refresh_entire_bucket_name_cache(self, name_id_iterable): self._buckets = dict(name_id_iterable) def get_bucket_id_or_none_from_bucket_name(self, bucket_name): return self._buckets.get(bucket_name) def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> str | None: for name, cached_id_ in self._buckets.items(): if cached_id_ == bucket_id: return name return None def list_bucket_names_ids(self) -> list[tuple[str, str]]: return [(name, id_) for name, id_ in self._buckets.items()] def save_bucket(self, bucket): self._buckets[bucket.name] = bucket.id_ def remove_bucket_name(self, bucket_name): if bucket_name in self._buckets: del self._buckets[bucket_name] @_raise_missing_if_result_is_none def get_account_id(self): return self._account_id @_raise_missing_if_result_is_none def get_application_key_id(self): return self._application_key_id @_raise_missing_if_result_is_none def get_account_auth_token(self): return self._auth_token @_raise_missing_if_result_is_none def get_api_url(self): return self._api_url @_raise_missing_if_result_is_none def get_application_key(self): return self._application_key @_raise_missing_if_result_is_none def get_download_url(self): return self._download_url @_raise_missing_if_result_is_none def get_recommended_part_size(self): return self._recommended_part_size @_raise_missing_if_result_is_none def get_absolute_minimum_part_size(self): return self._absolute_minimum_part_size @_raise_missing_if_result_is_none def get_realm(self): return self._realm @_raise_missing_if_result_is_none def get_allowed(self): return self._allowed @_raise_missing_if_result_is_none def get_s3_api_url(self): return self._s3_api_url b2-sdk-python-2.8.0/b2sdk/_internal/account_info/sqlite_account_info.py000066400000000000000000000572721474454370000262000ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/account_info/sqlite_account_info.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import json import logging import os import re import sqlite3 import stat import sys import threading from .exception import CorruptAccountInfo, MissingAccountData from .upload_url_pool import UrlPoolAccountInfo logger = logging.getLogger(__name__) B2_ACCOUNT_INFO_ENV_VAR = 'B2_ACCOUNT_INFO' B2_ACCOUNT_INFO_DEFAULT_FILE = os.path.join('~', '.b2_account_info') B2_ACCOUNT_INFO_PROFILE_FILE = os.path.join('~', '.b2db-{profile}.sqlite') B2_ACCOUNT_INFO_PROFILE_NAME_REGEXP = re.compile(r'[a-zA-Z0-9_\-]{1,64}') XDG_CONFIG_HOME_ENV_VAR = 'XDG_CONFIG_HOME' DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE = ( 5000000 # this value is used ONLY in migrating db, and in v1 wrapper, it is not ) # meant to be a default for other applications class SqliteAccountInfo(UrlPoolAccountInfo): """ Store account information in an `sqlite3 `_ database which is used to manage concurrent access to the data. The ``update_done`` table tracks the schema updates that have been completed. """ def __init__(self, file_name=None, last_upgrade_to_run=None, profile: str | None = None): """ Initialize SqliteAccountInfo. The exact algorithm used to determine the location of the database file is not API in any sense. If the location of the database file is required (for cleanup, etc), do not assume a specific resolution: instead, use ``self.filename`` to get the actual resolved location. SqliteAccountInfo currently checks locations in the following order: If ``profile`` arg is provided: * ``{B2_ACCOUNT_INFO_PROFILE_FILE}`` if it already exists * ``${XDG_CONFIG_HOME_ENV_VAR}/b2/db-.sqlite`` on XDG-compatible OSes (Linux, BSD) * ``{B2_ACCOUNT_INFO_PROFILE_FILE}`` Otherwise: * ``file_name``, if truthy * ``{B2_ACCOUNT_INFO_ENV_VAR}`` env var's value, if set * ``{B2_ACCOUNT_INFO_DEFAULT_FILE}``, if it already exists * ``${XDG_CONFIG_HOME_ENV_VAR}/b2/account_info`` on XDG-compatible OSes (Linux, BSD) * ``{B2_ACCOUNT_INFO_DEFAULT_FILE}``, as default If the directory ``${XDG_CONFIG_HOME_ENV_VAR}/b2`` does not exist (and is needed), it is created. :param str file_name: The sqlite file to use; overrides the default. :param int last_upgrade_to_run: For testing only, override the auto-update on the db. """ self.thread_local = threading.local() self.filename = self.get_user_account_info_path(file_name=file_name, profile=profile) logger.debug('%s file path to use: %s', self.__class__.__name__, self.filename) self._validate_database(last_upgrade_to_run) with self._get_connection() as conn: self._create_tables(conn, last_upgrade_to_run) super().__init__() # dirty trick to use parameters in the docstring if getattr(__init__, '__doc__', None): # don't break when using `python -oo` __init__.__doc__ = __init__.__doc__.format( **dict( B2_ACCOUNT_INFO_ENV_VAR=B2_ACCOUNT_INFO_ENV_VAR, B2_ACCOUNT_INFO_DEFAULT_FILE=B2_ACCOUNT_INFO_DEFAULT_FILE, B2_ACCOUNT_INFO_PROFILE_FILE=B2_ACCOUNT_INFO_PROFILE_FILE, XDG_CONFIG_HOME_ENV_VAR=XDG_CONFIG_HOME_ENV_VAR, ) ) @classmethod def _get_xdg_config_path(cls) -> str | None: """ Return XDG config path if the OS is XDG-compatible (Linux, BSD), None otherwise. If $XDG_CONFIG_HOME is empty but the OS is XDG compliant, fallback to ~/.config as expected by XDG standard. """ xdg_config_home = os.getenv(XDG_CONFIG_HOME_ENV_VAR) if xdg_config_home or sys.platform not in ('win32', 'darwin'): return xdg_config_home or os.path.join(os.path.expanduser('~/.config')) return None @classmethod def get_user_account_info_path(cls, file_name: str | None = None, profile: str | None = None): if profile and not B2_ACCOUNT_INFO_PROFILE_NAME_REGEXP.match(profile): raise ValueError(f'Invalid profile name: {profile}') profile_file = B2_ACCOUNT_INFO_PROFILE_FILE.format(profile=profile) if profile else None xdg_config_path = cls._get_xdg_config_path() if file_name: if profile: raise ValueError('Provide either file_name or profile, not both') user_account_info_path = file_name elif B2_ACCOUNT_INFO_ENV_VAR in os.environ: if profile: raise ValueError( f'Provide either {B2_ACCOUNT_INFO_ENV_VAR} env var or profile, not both' ) user_account_info_path = os.environ[B2_ACCOUNT_INFO_ENV_VAR] elif not profile and os.path.exists(os.path.expanduser(B2_ACCOUNT_INFO_DEFAULT_FILE)): user_account_info_path = B2_ACCOUNT_INFO_DEFAULT_FILE elif profile and os.path.exists(profile_file): user_account_info_path = profile_file elif xdg_config_path: os.makedirs(os.path.join(xdg_config_path, 'b2'), mode=0o755, exist_ok=True) file_name = f'db-{profile}.sqlite' if profile else 'account_info' user_account_info_path = os.path.join(xdg_config_path, 'b2', file_name) elif profile: user_account_info_path = profile_file else: user_account_info_path = B2_ACCOUNT_INFO_DEFAULT_FILE return os.path.expanduser(user_account_info_path) def _validate_database(self, last_upgrade_to_run=None): """ Make sure that the database is openable. Removes the file if it's not. """ # If there is no file there, that's fine. It will get created when # we connect. if not os.path.exists(self.filename): self._create_database(last_upgrade_to_run) return # If we can connect to the database, and do anything, then all is good. try: with self._connect() as conn: self._create_tables(conn, last_upgrade_to_run) return except sqlite3.DatabaseError: pass # fall through to next case # If the file contains JSON with the right stuff in it, convert from # the old representation. try: with open(self.filename, 'rb') as f: data = json.loads(f.read().decode('utf-8')) keys = [ 'account_id', 'application_key', 'account_auth_token', 'api_url', 'download_url', 'minimum_part_size', 'realm', ] if all(k in data for k in keys): # remove the json file os.unlink(self.filename) # create a database self._create_database(last_upgrade_to_run) # add the data from the JSON file with self._connect() as conn: self._create_tables(conn, last_upgrade_to_run) insert_statement = """ INSERT INTO account (account_id, application_key, account_auth_token, api_url, download_url, recommended_part_size, realm, absolute_minimum_part_size) values (?, ?, ?, ?, ?, ?, ?, ?); """ # Migrating from old schema is a little confusing, but the values change as: # minimum_part_size -> recommended_part_size # new column absolute_minimum_part_size = DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE conn.execute( insert_statement, (*(data[k] for k in keys), DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE), ) # all is happy now return except ValueError: # includes json.decoder.JSONDecodeError pass # Remove the corrupted file and create a new database raise CorruptAccountInfo(self.filename) def _get_connection(self): """ Connections to sqlite cannot be shared across threads. """ try: return self.thread_local.connection except AttributeError: self.thread_local.connection = self._connect() return self.thread_local.connection def _connect(self): return sqlite3.connect(self.filename, isolation_level='EXCLUSIVE') def _create_database(self, last_upgrade_to_run): """ Make sure that the database is created and has appropriate file permissions. This should be done before storing any sensitive data in it. """ # Prepare a file fd = os.open( self.filename, flags=os.O_RDWR | os.O_CREAT, mode=stat.S_IRUSR | stat.S_IWUSR, ) os.close(fd) # Create the tables in the database conn = self._connect() try: with conn: self._create_tables(conn, last_upgrade_to_run) finally: conn.close() def _create_tables(self, conn, last_upgrade_to_run): conn.execute( """ CREATE TABLE IF NOT EXISTS update_done ( update_number INT NOT NULL ); """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS account ( account_id TEXT NOT NULL, application_key TEXT NOT NULL, account_auth_token TEXT NOT NULL, api_url TEXT NOT NULL, download_url TEXT NOT NULL, minimum_part_size INT NOT NULL, realm TEXT NOT NULL ); """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS bucket ( bucket_name TEXT NOT NULL, bucket_id TEXT NOT NULL ); """ ) # This table is not used any more. We may use it again # someday if we save upload URLs across invocations of # the command-line tool. conn.execute( """ CREATE TABLE IF NOT EXISTS bucket_upload_url ( bucket_id TEXT NOT NULL, upload_url TEXT NOT NULL, upload_auth_token TEXT NOT NULL ); """ ) # By default, we run all the upgrades last_upgrade_to_run = 4 if last_upgrade_to_run is None else last_upgrade_to_run # Add the 'allowed' column if it hasn't been yet. if 1 <= last_upgrade_to_run: self._ensure_update(1, ['ALTER TABLE account ADD COLUMN allowed TEXT;']) # Add the 'account_id_or_app_key_id' column if it hasn't been yet if 2 <= last_upgrade_to_run: self._ensure_update( 2, ['ALTER TABLE account ADD COLUMN account_id_or_app_key_id TEXT;'] ) # Add the 's3_api_url' column if it hasn't been yet if 3 <= last_upgrade_to_run: self._ensure_update(3, ['ALTER TABLE account ADD COLUMN s3_api_url TEXT;']) if 4 <= last_upgrade_to_run: self._ensure_update( 4, [ f""" CREATE TABLE tmp_account ( account_id TEXT NOT NULL, application_key TEXT NOT NULL, account_auth_token TEXT NOT NULL, api_url TEXT NOT NULL, download_url TEXT NOT NULL, absolute_minimum_part_size INT NOT NULL DEFAULT {DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE}, recommended_part_size INT NOT NULL, realm TEXT NOT NULL, allowed TEXT, account_id_or_app_key_id TEXT, s3_api_url TEXT ); """, """INSERT INTO tmp_account( account_id, application_key, account_auth_token, api_url, download_url, recommended_part_size, realm, allowed, account_id_or_app_key_id, s3_api_url ) SELECT account_id, application_key, account_auth_token, api_url, download_url, minimum_part_size, realm, allowed, account_id_or_app_key_id, s3_api_url FROM account; """, 'DROP TABLE account;', """ CREATE TABLE account ( account_id TEXT NOT NULL, application_key TEXT NOT NULL, account_auth_token TEXT NOT NULL, api_url TEXT NOT NULL, download_url TEXT NOT NULL, absolute_minimum_part_size INT NOT NULL, recommended_part_size INT NOT NULL, realm TEXT NOT NULL, allowed TEXT, account_id_or_app_key_id TEXT, s3_api_url TEXT ); """, """INSERT INTO account( account_id, application_key, account_auth_token, api_url, download_url, absolute_minimum_part_size, recommended_part_size, realm, allowed, account_id_or_app_key_id, s3_api_url ) SELECT account_id, application_key, account_auth_token, api_url, download_url, absolute_minimum_part_size, recommended_part_size, realm, allowed, account_id_or_app_key_id, s3_api_url FROM tmp_account; """, 'DROP TABLE tmp_account;', ], ) def _ensure_update(self, update_number, update_commands: list[str]): """ Run the update with the given number if it hasn't been done yet. Does the update and stores the number as a single transaction, so they will always be in sync. """ with self._get_connection() as conn: conn.execute('BEGIN') cursor = conn.execute( 'SELECT COUNT(*) AS count FROM update_done WHERE update_number = ?;', (update_number,), ) update_count = cursor.fetchone()[0] if update_count == 0: for command in update_commands: conn.execute(command) conn.execute( 'INSERT INTO update_done (update_number) VALUES (?);', (update_number,) ) def clear(self): """ Remove all info about accounts and buckets. """ with self._get_connection() as conn: conn.execute('DELETE FROM account;') conn.execute('DELETE FROM bucket;') conn.execute('DELETE FROM bucket_upload_url;') def _set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id, ): assert self.allowed_is_valid(allowed) with self._get_connection() as conn: conn.execute('DELETE FROM account;') conn.execute('DELETE FROM bucket;') conn.execute('DELETE FROM bucket_upload_url;') insert_statement = """ INSERT INTO account (account_id, account_id_or_app_key_id, application_key, account_auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, realm, allowed, s3_api_url) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ conn.execute( insert_statement, ( account_id, application_key_id, application_key, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, realm, json.dumps(allowed), s3_api_url, ), ) def set_auth_data_with_schema_0_for_test( self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, ): """ Set authentication data for tests. :param str account_id: an account ID :param str auth_token: an authentication token :param str api_url: an API URL :param str download_url: a download URL :param int minimum_part_size: a minimum part size :param str application_key: an application key :param str realm: a realm to authorize account in """ with self._get_connection() as conn: conn.execute('DELETE FROM account;') conn.execute('DELETE FROM bucket;') conn.execute('DELETE FROM bucket_upload_url;') insert_statement = """ INSERT INTO account (account_id, application_key, account_auth_token, api_url, download_url, minimum_part_size, realm) values (?, ?, ?, ?, ?, ?, ?); """ conn.execute( insert_statement, ( account_id, application_key, auth_token, api_url, download_url, minimum_part_size, realm, ), ) def get_application_key(self): return self._get_account_info_or_raise('application_key') def get_account_id(self): return self._get_account_info_or_raise('account_id') def get_application_key_id(self): """ Return an application key ID. The 'account_id_or_app_key_id' column was not in the original schema, so it may be NULL. Nota bene - this is the only place where we are not renaming account_id_or_app_key_id to application_key_id because it requires a column change. application_key_id == account_id_or_app_key_id :rtype: str """ result = self._get_account_info_or_raise('account_id_or_app_key_id') if result is None: return self.get_account_id() else: return result def get_api_url(self): return self._get_account_info_or_raise('api_url') def get_account_auth_token(self): return self._get_account_info_or_raise('account_auth_token') def get_download_url(self): return self._get_account_info_or_raise('download_url') def get_realm(self): return self._get_account_info_or_raise('realm') def get_recommended_part_size(self): return self._get_account_info_or_raise('recommended_part_size') def get_absolute_minimum_part_size(self): return self._get_account_info_or_raise('absolute_minimum_part_size') def get_allowed(self): """ Return 'allowed' dictionary info. Example: .. code-block:: python { "bucketId": null, "bucketName": null, "capabilities": [ "listKeys", "writeKeys" ], "namePrefix": null } The 'allowed' column was not in the original schema, so it may be NULL. :rtype: dict """ allowed_json = self._get_account_info_or_raise('allowed') if allowed_json is None: return self.DEFAULT_ALLOWED else: return json.loads(allowed_json) def get_s3_api_url(self): result = self._get_account_info_or_raise('s3_api_url') if result is None: return '' else: return result def _get_account_info_or_raise(self, column_name): try: with self._get_connection() as conn: cursor = conn.execute(f'SELECT {column_name} FROM account;') value = cursor.fetchone()[0] return value except Exception as e: logger.exception( '_get_account_info_or_raise encountered a problem while trying to retrieve "%s"', column_name, ) raise MissingAccountData(str(e)) def refresh_entire_bucket_name_cache(self, name_id_iterable): with self._get_connection() as conn: conn.execute('DELETE FROM bucket;') for bucket_name, bucket_id in name_id_iterable: conn.execute( 'INSERT INTO bucket (bucket_name, bucket_id) VALUES (?, ?);', (bucket_name, bucket_id), ) def save_bucket(self, bucket): with self._get_connection() as conn: conn.execute('DELETE FROM bucket WHERE bucket_id = ?;', (bucket.id_,)) conn.execute( 'INSERT INTO bucket (bucket_id, bucket_name) VALUES (?, ?);', (bucket.id_, bucket.name), ) def remove_bucket_name(self, bucket_name): with self._get_connection() as conn: conn.execute('DELETE FROM bucket WHERE bucket_name = ?;', (bucket_name,)) def get_bucket_id_or_none_from_bucket_name(self, bucket_name): return self._safe_query( 'SELECT bucket_id FROM bucket WHERE bucket_name = ?;', (bucket_name,) ) def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> str | None: return self._safe_query('SELECT bucket_name FROM bucket WHERE bucket_id = ?;', (bucket_id,)) def list_bucket_names_ids(self) -> list[tuple[str, str]]: with self._get_connection() as conn: cursor = conn.execute('SELECT bucket_name, bucket_id FROM bucket;') return cursor.fetchall() def _safe_query(self, query, params): try: with self._get_connection() as conn: cursor = conn.execute(query, params) return cursor.fetchone()[0] except TypeError: # TypeError: 'NoneType' object is unsubscriptable return None except sqlite3.Error: return None b2-sdk-python-2.8.0/b2sdk/_internal/account_info/stub.py000066400000000000000000000103611474454370000231110ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/account_info/stub.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import collections import threading from .abstract import AbstractAccountInfo class StubAccountInfo(AbstractAccountInfo): def __init__(self): self._clear_stub_account_fields() def clear(self): self._clear_stub_account_fields() def _clear_stub_account_fields(self): self.application_key = None self.buckets = {} self.account_id = None self.allowed = None self.api_url = None self.auth_token = None self.download_url = None self.absolute_minimum_part_size = None self.recommended_part_size = None self.realm = None self.application_key_id = None self._large_file_uploads = collections.defaultdict(list) self._large_file_uploads_lock = threading.Lock() def clear_bucket_upload_data(self, bucket_id): if bucket_id in self.buckets: del self.buckets[bucket_id] def _set_auth_data( self, account_id, auth_token, api_url, download_url, recommended_part_size, absolute_minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id, ): self.account_id = account_id self.auth_token = auth_token self.api_url = api_url self.download_url = download_url self.recommended_part_size = recommended_part_size self.absolute_minimum_part_size = absolute_minimum_part_size self.application_key = application_key self.realm = realm self.s3_api_url = s3_api_url self.allowed = allowed self.application_key_id = application_key_id def refresh_entire_bucket_name_cache(self, name_id_iterable): self.buckets = {} def get_bucket_id_or_none_from_bucket_name(self, bucket_name): return None def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> str | None: return None def list_bucket_names_ids(self) -> list[tuple[str, str]]: return list((bucket.bucket_name, bucket.id_) for bucket in self.buckets.values()) def save_bucket(self, bucket): self.buckets[bucket.id_] = bucket def remove_bucket_name(self, bucket_name): pass def take_bucket_upload_url(self, bucket_id): return (None, None) def put_bucket_upload_url(self, bucket_id, upload_url, upload_auth_token): pass def get_account_id(self): return self.account_id def get_application_key_id(self): return self.application_key_id def get_account_auth_token(self): return self.auth_token def get_api_url(self): return self.api_url def get_application_key(self): return self.application_key def get_download_url(self): return self.download_url def get_recommended_part_size(self): return self.recommended_part_size def get_absolute_minimum_part_size(self): return self.absolute_minimum_part_size def get_realm(self): return self.realm def get_allowed(self): return self.allowed def get_bucket_upload_data(self, bucket_id): return self.buckets.get(bucket_id, (None, None)) def get_s3_api_url(self): return self.s3_api_url def put_large_file_upload_url(self, file_id, upload_url, upload_auth_token): with self._large_file_uploads_lock: self._large_file_uploads[file_id].append((upload_url, upload_auth_token)) def take_large_file_upload_url(self, file_id): with self._large_file_uploads_lock: upload_urls = self._large_file_uploads.get(file_id, []) if not upload_urls: return (None, None) else: return upload_urls.pop() def clear_large_file_upload_urls(self, file_id): with self._large_file_uploads_lock: if file_id in self._large_file_uploads: del self._large_file_uploads[file_id] b2-sdk-python-2.8.0/b2sdk/_internal/account_info/upload_url_pool.py000066400000000000000000000067171474454370000253450ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/account_info/upload_url_pool.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import collections import threading from abc import abstractmethod from .abstract import AbstractAccountInfo class UploadUrlPool: """ For each key (either a bucket id or large file id), hold a pool of (url, auth_token) pairs. .. note: This class is thread-safe. """ def __init__(self): self._lock = threading.Lock() self._pool = collections.defaultdict(list) def put(self, key, url, auth_token): """ Add the url and auth token to the pool for the given key. :param str key: bucket ID or large file ID :param str url: bucket or file URL :param str auth_token: authentication token """ with self._lock: pair = (url, auth_token) self._pool[key].append(pair) def take(self, key): """ Return a (url, auth_token) if one is available, or (None, None) if not. :param str key: bucket ID or large file ID :rtype: tuple """ with self._lock: pair_list = self._pool[key] if pair_list: return pair_list.pop() else: return (None, None) def clear_for_key(self, key): """ Remove an item from the pool by key. :param str key: bucket ID or large file ID """ with self._lock: if key in self._pool: del self._pool[key] class UrlPoolAccountInfo(AbstractAccountInfo): """ Implement part of :py:class:`AbstractAccountInfo` for upload URL pool management with a simple, key-value storage, such as :py:class:`b2sdk.v2.UploadUrlPool`. """ # staticmethod is necessary here to avoid the first argument binding to the first argument (like ``partial(fun, arg)``) BUCKET_UPLOAD_POOL_CLASS = staticmethod( UploadUrlPool ) #: A url pool class to use for small files. LARGE_FILE_UPLOAD_POOL_CLASS = staticmethod( UploadUrlPool ) #: A url pool class to use for large files. def __init__(self): super().__init__() self._reset_upload_pools() @abstractmethod def clear(self): self._reset_upload_pools() return super().clear() def _reset_upload_pools(self): self._bucket_uploads = self.BUCKET_UPLOAD_POOL_CLASS() self._large_file_uploads = self.LARGE_FILE_UPLOAD_POOL_CLASS() # bucket upload url def put_bucket_upload_url(self, bucket_id, upload_url, upload_auth_token): self._bucket_uploads.put(bucket_id, upload_url, upload_auth_token) def clear_bucket_upload_data(self, bucket_id): self._bucket_uploads.clear_for_key(bucket_id) def take_bucket_upload_url(self, bucket_id): return self._bucket_uploads.take(bucket_id) # large file upload url def put_large_file_upload_url(self, file_id, upload_url, upload_auth_token): self._large_file_uploads.put(file_id, upload_url, upload_auth_token) def take_large_file_upload_url(self, file_id): return self._large_file_uploads.take(file_id) def clear_large_file_upload_urls(self, file_id): self._large_file_uploads.clear_for_key(file_id) b2-sdk-python-2.8.0/b2sdk/_internal/api.py000066400000000000000000000632571474454370000202520ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/api.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from collections.abc import Sequence from contextlib import suppress from typing import Generator from .account_info.abstract import AbstractAccountInfo from .account_info.exception import MissingAccountData from .api_config import DEFAULT_HTTP_API_CONFIG, B2HttpApiConfig from .application_key import ApplicationKey, BaseApplicationKey, FullApplicationKey from .bucket import Bucket, BucketFactory from .cache import AbstractCache from .encryption.setting import EncryptionSetting from .exception import ( BucketIdNotFound, NonExistentBucket, RestrictedBucket, RestrictedBucketMissing, ) from .file_lock import FileRetentionSetting, LegalHold from .file_version import ( DownloadVersion, DownloadVersionFactory, FileIdAndName, FileVersion, FileVersionFactory, ) from .large_file.services import LargeFileServices from .progress import AbstractProgressListener from .raw_api import API_VERSION, LifecycleRule from .replication.setting import ReplicationConfiguration from .session import B2Session from .transfer import ( CopyManager, DownloadManager, Emerger, UploadManager, ) from .transfer.inbound.downloaded_file import DownloadedFile from .utils import B2TraceMeta, b2_url_encode, limit_trace_arguments logger = logging.getLogger(__name__) def url_for_api(info, api_name): """ Return URL for an API endpoint. :param info: account info :param str api_nam: :rtype: str """ if api_name in ['b2_download_file_by_id']: base = info.get_download_url() else: base = info.get_api_url() return f'{base}/b2api/{API_VERSION}/{api_name}' class Services: """Gathers objects that provide high level logic over raw api usage.""" UPLOAD_MANAGER_CLASS = staticmethod(UploadManager) COPY_MANAGER_CLASS = staticmethod(CopyManager) DOWNLOAD_MANAGER_CLASS = staticmethod(DownloadManager) LARGE_FILE_SERVICES_CLASS = staticmethod(LargeFileServices) def __init__( self, api, max_upload_workers: int | None = None, max_copy_workers: int | None = None, max_download_workers: int | None = None, save_to_buffer_size: int | None = None, check_download_hash: bool = True, max_download_streams_per_file: int | None = None, ): """ Initialize Services object using given session. :param b2sdk.v2.B2Api api: :param max_upload_workers: a number of upload threads :param max_copy_workers: a number of copy threads :param max_download_workers: maximum number of download threads :param save_to_buffer_size: buffer size to use when writing files using DownloadedFile.save_to :param check_download_hash: whether to check hash of downloaded files. Can be disabled for files with internal checksums, for example, or to forcefully retrieve objects with corrupted payload or hash value :param max_download_streams_per_file: how many streams to use for parallel downloader """ self.api = api self.session = api.session self.large_file = self.LARGE_FILE_SERVICES_CLASS(self) self.upload_manager = self.UPLOAD_MANAGER_CLASS( services=self, max_workers=max_upload_workers ) self.copy_manager = self.COPY_MANAGER_CLASS(services=self, max_workers=max_copy_workers) assert max_download_streams_per_file is None or max_download_streams_per_file >= 1 self.download_manager = self.DOWNLOAD_MANAGER_CLASS( services=self, max_workers=max_download_workers, write_buffer_size=save_to_buffer_size, check_hash=check_download_hash, max_download_streams_per_file=max_download_streams_per_file, ) self.emerger = Emerger(self) class B2Api(metaclass=B2TraceMeta): """ Provide file-level access to B2 services. While :class:`b2sdk.v2.B2RawHTTPApi` provides direct access to the B2 web APIs, this class handles several things that simplify the task of uploading and downloading files: - re-acquires authorization tokens when they expire - retrying uploads when an upload URL is busy - breaking large files into parts - emulating a directory structure (B2 buckets are flat) Adds an object-oriented layer on top of the raw API, so that buckets and files returned are Python objects with accessor methods. The class also keeps a cache of information needed to access the service, such as auth tokens and upload URLs. """ BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) BUCKET_CLASS = staticmethod(Bucket) SESSION_CLASS = staticmethod(B2Session) FILE_VERSION_FACTORY_CLASS = staticmethod(FileVersionFactory) DOWNLOAD_VERSION_FACTORY_CLASS = staticmethod(DownloadVersionFactory) SERVICES_CLASS = staticmethod(Services) DEFAULT_LIST_KEY_COUNT = 1000 def __init__( self, account_info: AbstractAccountInfo | None = None, cache: AbstractCache | None = None, max_upload_workers: int | None = None, max_copy_workers: int | None = None, api_config: B2HttpApiConfig = DEFAULT_HTTP_API_CONFIG, max_download_workers: int | None = None, save_to_buffer_size: int | None = None, check_download_hash: bool = True, max_download_streams_per_file: int | None = None, ): """ Initialize the API using the given account info. :param account_info: To learn more about Account Info objects, see here :class:`~b2sdk.v2.SqliteAccountInfo` :param cache: It is used by B2Api to cache the mapping between bucket name and bucket ids. default is :class:`~b2sdk._internal.cache.DummyCache` :param max_upload_workers: a number of upload threads :param max_copy_workers: a number of copy threads :param api_config: :param max_download_workers: maximum number of download threads :param save_to_buffer_size: buffer size to use when writing files using DownloadedFile.save_to :param check_download_hash: whether to check hash of downloaded files. Can be disabled for files with internal checksums, for example, or to forcefully retrieve objects with corrupted payload or hash value :param max_download_streams_per_file: number of streams for parallel download manager """ self.session = self.SESSION_CLASS( account_info=account_info, cache=cache, api_config=api_config ) self.api_config = api_config self.file_version_factory = self.FILE_VERSION_FACTORY_CLASS(self) self.download_version_factory = self.DOWNLOAD_VERSION_FACTORY_CLASS(self) self.services = self.SERVICES_CLASS( api=self, max_upload_workers=max_upload_workers, max_copy_workers=max_copy_workers, max_download_workers=max_download_workers, save_to_buffer_size=save_to_buffer_size, check_download_hash=check_download_hash, max_download_streams_per_file=max_download_streams_per_file, ) @property def account_info(self): return self.session.account_info @property def cache(self): return self.session.cache @property def raw_api(self): """ .. warning:: :class:`~b2sdk._internal.raw_api.B2RawHTTPApi` attribute is deprecated. :class:`~b2sdk._internal.session.B2Session` expose all :class:`~b2sdk._internal.raw_api.B2RawHTTPApi` methods now. """ return self.session.raw_api def authorize_automatically(self): """ Perform automatic account authorization, retrieving all account data from account info object passed during initialization. """ return self.session.authorize_automatically() @limit_trace_arguments(only=('self', 'realm')) def authorize_account(self, application_key_id, application_key, realm='production'): """ Perform account authorization. :param str application_key_id: :term:`application key ID` :param str application_key: user's :term:`application key` :param str realm: a realm to authorize account in (usually just "production") """ self.session.authorize_account(realm, application_key_id, application_key) self._populate_bucket_cache_from_key() def get_account_id(self): """ Return the account ID. :rtype: str """ return self.account_info.get_account_id() # buckets def create_bucket( self, name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, default_server_side_encryption: EncryptionSetting | None = None, is_file_lock_enabled: bool | None = None, replication: ReplicationConfiguration | None = None, ) -> Bucket: """ Create a bucket. :param str name: bucket name :param str bucket_type: a bucket type, could be one of the following values: ``"allPublic"``, ``"allPrivate"`` :param dict bucket_info: additional bucket info to store with the bucket :param dict cors_rules: bucket CORS rules to store with the bucket :param lifecycle_rules: bucket lifecycle rules to store with the bucket :param b2sdk.v2.EncryptionSetting default_server_side_encryption: default server side encryption settings (``None`` if unknown) :param bool is_file_lock_enabled: boolean value specifies whether bucket is File Lock-enabled :param b2sdk.v2.ReplicationConfiguration replication: bucket replication rules or ``None`` :return: a Bucket object :rtype: b2sdk.v2.Bucket """ account_id = self.account_info.get_account_id() response = self.session.create_bucket( account_id, name, bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, default_server_side_encryption=default_server_side_encryption, is_file_lock_enabled=is_file_lock_enabled, replication=replication, ) bucket = self.BUCKET_FACTORY_CLASS.from_api_bucket_dict(self, response) assert ( name == bucket.name ), f'API created a bucket with different name than requested: {name} != {name}' assert ( bucket_type == bucket.type_ ), f'API created a bucket with different type than requested: {bucket_type} != {bucket.type_}' self.cache.save_bucket(bucket) return bucket def download_file_by_id( self, file_id: str, progress_listener: AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: EncryptionSetting | None = None, ) -> DownloadedFile: """ Download a file with the given ID. :param str file_id: a file ID :param progress_listener: a progress listener object to use, or ``None`` to not track progress :param range_: a list of two integers, the first one is a start\ position, and the second one is the end position in the file :param encryption: encryption settings (``None`` if unknown) """ url = self.session.get_download_url_by_id(file_id) return self.services.download_manager.download_file_from_url( url, progress_listener, range_, encryption, ) def update_file_retention( self, file_id: str, file_name: str, file_retention: FileRetentionSetting, bypass_governance: bool = False, ) -> FileRetentionSetting: return FileRetentionSetting.from_server_response( self.session.update_file_retention( file_id, file_name, file_retention, bypass_governance, ) ) def update_file_legal_hold( self, file_id: str, file_name: str, legal_hold: LegalHold, ) -> LegalHold: return LegalHold.from_server_response( self.session.update_file_legal_hold( file_id, file_name, legal_hold, ) ) def get_bucket_by_id(self, bucket_id: str) -> Bucket: """ Return the Bucket matching the given bucket_id. :raises b2sdk.v2.exception.BucketIdNotFound: if the bucket does not exist in the account """ # Give a useful warning if the current application key does not # allow access to bucket. self.check_bucket_id_restrictions(bucket_id) # First, try the cache. bucket_name = self.cache.get_bucket_name_or_none_from_bucket_id(bucket_id) if bucket_name is not None: return self.BUCKET_CLASS(self, bucket_id, name=bucket_name) # Second, ask the service for bucket in self.list_buckets(bucket_id=bucket_id): assert bucket.id_ == bucket_id return bucket # There is no such bucket. raise BucketIdNotFound(bucket_id) def get_bucket_by_name(self, bucket_name: str) -> Bucket: """ Return the Bucket matching the given bucket_name. :param str bucket_name: the name of the bucket to return :return: a Bucket object :rtype: b2sdk.v2.Bucket :raises b2sdk.v2.exception.NonExistentBucket: if the bucket does not exist in the account """ # Give a useful warning if the current application key does not # allow access to the named bucket. self.check_bucket_name_restrictions(bucket_name) # First, try the cache. id_ = self.cache.get_bucket_id_or_none_from_bucket_name(bucket_name) if id_ is not None: return self.BUCKET_CLASS(self, id_, name=bucket_name) # Second, ask the service for bucket in self.list_buckets(bucket_name=bucket_name): assert bucket.name.lower() == bucket_name.lower() return bucket # There is no such bucket. raise NonExistentBucket(bucket_name) def delete_bucket(self, bucket): """ Delete a chosen bucket. :param b2sdk.v2.Bucket bucket: a :term:`bucket` to delete :rtype: None """ account_id = self.account_info.get_account_id() self.session.delete_bucket(account_id, bucket.id_) def list_buckets( self, bucket_name=None, bucket_id=None, *, use_cache: bool = False ) -> Sequence[Bucket]: """ Call ``b2_list_buckets`` and return a list of buckets. When no bucket name nor ID is specified, returns *all* of the buckets in the account. When a bucket name or ID is given, returns just that bucket. When authorized with an :term:`application key` restricted to one :term:`bucket`, you must specify the bucket name or bucket id, or the request will be unauthorized. :param str bucket_name: the name of the one bucket to return :param str bucket_id: the ID of the one bucket to return :param bool use_cache: if ``True`` use cached bucket list if available and not empty :rtype: list[b2sdk.v2.Bucket] """ # Give a useful warning if the current application key does not # allow access to the named bucket. if bucket_name is not None and bucket_id is not None: raise ValueError('Provide either bucket_name or bucket_id, not both') if bucket_id: self.check_bucket_id_restrictions(bucket_id) else: self.check_bucket_name_restrictions(bucket_name) if use_cache: cached_list = self.cache.list_bucket_names_ids() buckets = [ self.BUCKET_CLASS(self, cache_b_id, name=cached_b_name) for cached_b_name, cache_b_id in cached_list if ( (bucket_name is None or bucket_name == cached_b_name) and (bucket_id is None or bucket_id == cache_b_id) ) ] if buckets: logger.debug('Using cached bucket list as it is not empty') return buckets account_id = self.account_info.get_account_id() response = self.session.list_buckets( account_id, bucket_name=bucket_name, bucket_id=bucket_id ) buckets = self.BUCKET_FACTORY_CLASS.from_api_response(self, response) if bucket_name or bucket_id: # If a bucket_name or bucket_id is specified we don't clear the cache because the other buckets could still # be valid. So we save the one bucket returned from the list_buckets call. for bucket in buckets: self.cache.save_bucket(bucket) else: # Otherwise we want to clear the cache and save the buckets returned from list_buckets # since we just got a new list of all the buckets for this account. self.cache.set_bucket_name_cache(buckets) return buckets def list_parts(self, file_id, start_part_number=None, batch_size=None): """ Generator that yields a :py:class:`b2sdk.v2.Part` for each of the parts that have been uploaded. :param str file_id: the ID of the large file that is not finished :param int start_part_number: the first part number to return; defaults to the first part :param int batch_size: the number of parts to fetch at a time from the server :rtype: generator """ return self.services.large_file.list_parts( file_id, start_part_number=start_part_number, batch_size=batch_size ) # delete/cancel def cancel_large_file(self, file_id: str) -> FileIdAndName: """ Cancel a large file upload. """ return self.services.large_file.cancel_large_file(file_id) def delete_file_version( self, file_id: str, file_name: str, bypass_governance: bool = False ) -> FileIdAndName: """ Permanently and irrevocably delete one version of a file. bypass_governance must be set to true if deleting a file version protected by Object Lock governance mode retention settings (unless its retention period expired) """ # filename argument is not first, because one day it may become optional response = self.session.delete_file_version(file_id, file_name, bypass_governance) return FileIdAndName.from_cancel_or_delete_response(response) # download def get_download_url_for_fileid(self, file_id): """ Return a URL to download the given file by ID. :param str file_id: a file ID """ url = url_for_api(self.account_info, 'b2_download_file_by_id') return f'{url}?fileId={file_id}' def get_download_url_for_file_name(self, bucket_name, file_name): """ Return a URL to download the given file by name. :param str bucket_name: a bucket name :param str file_name: a file name """ self.check_bucket_name_restrictions(bucket_name) return ( f'{self.account_info.get_download_url()}/file/{bucket_name}/{b2_url_encode(file_name)}' ) # keys def create_key( self, capabilities: list[str], key_name: str, valid_duration_seconds: int | None = None, bucket_id: str | None = None, name_prefix: str | None = None, ) -> FullApplicationKey: """ Create a new :term:`application key`. :param capabilities: a list of capabilities :param key_name: a name of a key :param valid_duration_seconds: key auto-expire time after it is created, in seconds, or ``None`` to not expire :param bucket_id: a bucket ID to restrict the key to, or ``None`` to not restrict :param name_prefix: a remote filename prefix to restrict the key to or ``None`` to not restrict """ account_id = self.account_info.get_account_id() response = self.session.create_key( account_id, capabilities=capabilities, key_name=key_name, valid_duration_seconds=valid_duration_seconds, bucket_id=bucket_id, name_prefix=name_prefix, ) assert set(response['capabilities']) == set(capabilities) assert response['keyName'] == key_name return FullApplicationKey.from_create_response(response) def delete_key(self, application_key: BaseApplicationKey): """ Delete :term:`application key`. :param application_key: an :term:`application key` """ return self.delete_key_by_id(application_key.id_) def delete_key_by_id(self, application_key_id: str) -> ApplicationKey: """ Delete :term:`application key`. :param application_key_id: an :term:`application key ID` """ response = self.session.delete_key(application_key_id=application_key_id) return ApplicationKey.from_api_response(response) def list_keys( self, start_application_key_id: str | None = None ) -> Generator[ApplicationKey, None, None]: """ List application keys. Lazily perform requests to B2 cloud and return all keys. :param start_application_key_id: an :term:`application key ID` to start from or ``None`` to start from the beginning """ account_id = self.account_info.get_account_id() while True: response = self.session.list_keys( account_id, max_key_count=self.DEFAULT_LIST_KEY_COUNT, start_application_key_id=start_application_key_id, ) for entry in response['keys']: yield ApplicationKey.from_api_response(entry) next_application_key_id = response['nextApplicationKeyId'] if next_application_key_id is None: return start_application_key_id = next_application_key_id def get_key(self, key_id: str) -> ApplicationKey | None: """ Gets information about a single key: it's capabilities, prefix, name etc Returns `None` if the key does not exist. Raises an exception if profile is not permitted to list keys. """ with suppress(StopIteration): key = next(self.list_keys(start_application_key_id=key_id)) # list_keys() may return some other key if `key_id` does not exist; # thus manually check that we retrieved the right key if key.id_ == key_id: return key # other def get_file_info(self, file_id: str) -> FileVersion: """ Gets info about file version. :param str file_id: the id of the file whose info will be retrieved. """ return self.file_version_factory.from_api_response( self.session.get_file_info_by_id(file_id) ) def get_file_info_by_name(self, bucket_name: str, file_name: str) -> DownloadVersion: """ Gets info about a file version. Similar to `get_file_info` but takes the bucket name and file name instead of file id. :param str bucket_name: The name of the bucket where the file resides. :param str file_name: The name of the file whose info will be retrieved. """ bucket = self.get_bucket_by_name(bucket_name) return bucket.get_file_info_by_name(file_name) def check_bucket_name_restrictions(self, bucket_name: str): """ Check to see if the allowed field from authorize-account has a bucket restriction. If it does, checks if the bucket_name for a given api call matches that. If not, it raises a :py:exc:`b2sdk.v2.exception.RestrictedBucket` error. :raises b2sdk.v2.exception.RestrictedBucket: if the account is not allowed to use this bucket """ self._check_bucket_restrictions('bucketName', bucket_name) def check_bucket_id_restrictions(self, bucket_id: str): """ Check to see if the allowed field from authorize-account has a bucket restriction. If it does, checks if the bucket_id for a given api call matches that. If not, it raises a :py:exc:`b2sdk.v2.exception.RestrictedBucket` error. :raises b2sdk.v2.exception.RestrictedBucket: if the account is not allowed to use this bucket """ self._check_bucket_restrictions('bucketId', bucket_id) def _check_bucket_restrictions(self, key, value): allowed = self.account_info.get_allowed() allowed_bucket_identifier = allowed[key] if allowed_bucket_identifier is not None: if allowed_bucket_identifier != value: raise RestrictedBucket(allowed_bucket_identifier) def _populate_bucket_cache_from_key(self): # If the key is restricted to the bucket, pre-populate the cache with it try: allowed = self.account_info.get_allowed() except MissingAccountData: return allowed_bucket_id = allowed.get('bucketId') if allowed_bucket_id is None: return allowed_bucket_name = allowed.get('bucketName') # If we have bucketId set we still need to check bucketName. If the bucketName is None, # it means that the bucketId belongs to a bucket that was already removed. if allowed_bucket_name is None: raise RestrictedBucketMissing() self.cache.save_bucket(self.BUCKET_CLASS(self, allowed_bucket_id, name=allowed_bucket_name)) b2-sdk-python-2.8.0/b2sdk/_internal/api_config.py000066400000000000000000000033241474454370000215640ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/api_config.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from typing import Callable import requests from .raw_api import AbstractRawApi, B2RawHTTPApi class B2HttpApiConfig: DEFAULT_RAW_API_CLASS = B2RawHTTPApi def __init__( self, http_session_factory: Callable[[], requests.Session] = requests.Session, install_clock_skew_hook: bool = True, user_agent_append: str | None = None, _raw_api_class: type[AbstractRawApi] | None = None, decode_content: bool = False, ): """ A structure with params to be passed to low level API. :param http_session_factory: a callable that returns a requests.Session object (or a compatible one) :param install_clock_skew_hook: if True, install a clock skew hook :param user_agent_append: if provided, the string will be appended to the User-Agent :param _raw_api_class: AbstractRawApi-compliant class :param decode_content: If true, the underlying http backend will try to decode encoded files when downloading, based on the response headers """ self.http_session_factory = http_session_factory self.install_clock_skew_hook = install_clock_skew_hook self.user_agent_append = user_agent_append self.raw_api_class = _raw_api_class or self.DEFAULT_RAW_API_CLASS self.decode_content = decode_content DEFAULT_HTTP_API_CONFIG = B2HttpApiConfig() b2-sdk-python-2.8.0/b2sdk/_internal/application_key.py000066400000000000000000000132531474454370000226430ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/application_key.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations class BaseApplicationKey: """Common methods for ApplicationKey and FullApplicationKey.""" def __init__( self, key_name: str, application_key_id: str, capabilities: list[str], account_id: str, expiration_timestamp_millis: int | None = None, bucket_id: str | None = None, name_prefix: str | None = None, options: list[str] | None = None, ): """ :param key_name: name of the key, assigned by user :param application_key_id: key id, used to authenticate :param capabilities: list of capabilities assigned to this key :param account_id: account's id :param expiration_timestamp_millis: expiration time of the key :param bucket_id: if restricted to a bucket, this is the bucket's id :param name_prefix: if restricted to some files, this is their prefix :param options: reserved for future use """ self.key_name = key_name self.id_ = application_key_id self.capabilities = capabilities self.account_id = account_id self.expiration_timestamp_millis = expiration_timestamp_millis self.bucket_id = bucket_id self.name_prefix = name_prefix self.options = options @classmethod def parse_response_dict(cls, response: dict): mandatory_args = { 'key_name': response['keyName'], 'application_key_id': response['applicationKeyId'], 'capabilities': response['capabilities'], 'account_id': response['accountId'], } optional_args = { 'expiration_timestamp_millis': response.get('expirationTimestamp'), 'bucket_id': response.get('bucketId'), 'name_prefix': response.get('namePrefix'), 'options': response.get('options'), } return { **mandatory_args, **{key: value for key, value in optional_args.items() if value is not None}, } def has_capabilities(self, capabilities) -> bool: """checks whether the key has ALL of the given capabilities""" return len(set(capabilities) - set(self.capabilities)) == 0 def as_dict(self): """Represent the key as a dict, like the one returned by B2 cloud""" mandatory_keys = { 'keyName': self.key_name, 'applicationKeyId': self.id_, 'capabilities': self.capabilities, 'accountId': self.account_id, } optional_keys = { 'expirationTimestamp': self.expiration_timestamp_millis, 'bucketId': self.bucket_id, 'namePrefix': self.name_prefix, 'options': self.options, } return { **mandatory_keys, **{key: value for key, value in optional_keys.items() if value is not None}, } class ApplicationKey(BaseApplicationKey): """Dataclass for storing info about an application key returned by delete-key or list-keys.""" @classmethod def from_api_response(cls, response: dict) -> ApplicationKey: """Create an ApplicationKey object from a delete-key or list-key response (a parsed json object).""" return cls(**cls.parse_response_dict(response)) class FullApplicationKey(BaseApplicationKey): """Dataclass for storing info about an application key, including the actual key, as returned by create-key.""" def __init__( self, key_name: str, application_key_id: str, application_key: str, capabilities: list[str], account_id: str, expiration_timestamp_millis: int | None = None, bucket_id: str | None = None, name_prefix: str | None = None, options: list[str] | None = None, ): """ :param key_name: name of the key, assigned by user :param application_key_id: key id, used to authenticate :param application_key: the actual secret key :param capabilities: list of capabilities assigned to this key :param account_id: account's id :param expiration_timestamp_millis: expiration time of the key :param bucket_id: if restricted to a bucket, this is the bucket's id :param name_prefix: if restricted to some files, this is their prefix :param options: reserved for future use """ self.application_key = application_key super().__init__( key_name=key_name, application_key_id=application_key_id, capabilities=capabilities, account_id=account_id, expiration_timestamp_millis=expiration_timestamp_millis, bucket_id=bucket_id, name_prefix=name_prefix, options=options, ) @classmethod def from_create_response(cls, response: dict) -> FullApplicationKey: """Create a FullApplicationKey object from a create-key response (a parsed json object).""" return cls(**cls.parse_response_dict(response)) @classmethod def parse_response_dict(cls, response: dict): result = super().parse_response_dict(response) result['application_key'] = response['applicationKey'] return result def as_dict(self): """Represent the key as a dict, like the one returned by B2 cloud""" return { **super().as_dict(), 'applicationKey': self.application_key, } b2-sdk-python-2.8.0/b2sdk/_internal/b2http.py000066400000000000000000000533621474454370000207000ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/b2http.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import datetime import io import json import locale import logging import socket import threading import time from contextlib import contextmanager from random import random from typing import Any, Callable try: from typing_extensions import Literal except ImportError: from typing import Literal import requests from requests.adapters import HTTPAdapter from b2sdk.version import USER_AGENT from .api_config import DEFAULT_HTTP_API_CONFIG, B2HttpApiConfig from .exception import ( B2ConnectionError, B2Error, B2RequestTimeout, B2RequestTimeoutDuringUpload, BadDateFormat, BrokenPipe, ClockSkew, ConnectionReset, PotentialS3EndpointPassedAsRealm, UnknownError, UnknownHost, interpret_b2_error, ) from .requests import NotDecompressingResponse from .utils.typing import JSON LOCALE_LOCK = threading.Lock() logger = logging.getLogger(__name__) def _print_exception(e, indent=''): """ Used for debugging to print out nested exception structures. :param str indent: message prefix """ print(indent + 'EXCEPTION', repr(e)) print(indent + 'CLASS', type(e)) for i, a in enumerate(e.args): print(indent + 'ARG %d: %s' % (i, repr(a))) if isinstance(a, Exception): _print_exception(a, indent + ' ') @contextmanager def setlocale(name): with LOCALE_LOCK: saved = locale.setlocale(locale.LC_ALL) try: yield locale.setlocale(locale.LC_ALL, name) finally: locale.setlocale(locale.LC_ALL, saved) class ResponseContextManager: """ A context manager that closes a requests.Response when done. """ def __init__(self, response): self.response = response def __enter__(self): return self.response def __exit__(self, exc_type, exc_val, exc_tb): return None class HttpCallback: """ A callback object that does nothing. Overrides pre_request and/or post_request as desired. """ def pre_request(self, method, url, headers): """ Called before processing an HTTP request. Raises an exception if this request should not be processed. The exception raised must inherit from B2HttpCallbackPreRequestException. :param str method: str, one of: 'POST', 'GET', etc. :param str url: the URL that will be used :param dict headers: the header sent with the request """ def post_request(self, method, url, headers, response): """ Called after processing an HTTP request. Should not raise an exception. Raises an exception if this request should be treated as failing. The exception raised must inherit from B2HttpCallbackPostRequestException. :param str method: one of: 'POST', 'GET', etc. :param str url: the URL that will be used :param dict headers: the header sent with the request :param response: a response object from the requests library """ class ClockSkewHook(HttpCallback): def post_request(self, method, url, headers, response): """ Raise an exception if the clock in the server is too different from the clock on the local host. The Date header contains a string that looks like: "Fri, 16 Dec 2016 20:52:30 GMT". :param str method: one of: 'POST', 'GET', etc. :param str url: the URL that will be used :param dict headers: the header sent with the request :param response: a response object from the requests library """ # Make a string that uses month numbers instead of month names server_date_str = response.headers['Date'] # Convert the server time to a datetime object try: with setlocale('C'): # "%Z" always creates naive datetimes, even though the timezone # is specified. https://github.com/python/cpython/issues/76678 # Anyway, thankfully, HTTP/1.1 spec requires the string # to always say "GMT", and provide UTC time. # https://datatracker.ietf.org/doc/html/rfc2616#section-3.3.1 server_time = datetime.datetime.strptime( server_date_str, '%a, %d %b %Y %H:%M:%S GMT' ).replace(tzinfo=datetime.timezone.utc) except ValueError: logger.exception('server returned date in an inappropriate format') raise BadDateFormat(server_date_str) # Get the local time local_time = datetime.datetime.now(datetime.timezone.utc) # Check the difference. max_allowed = 10 * 60 # ten minutes, in seconds skew = local_time - server_time skew_seconds = skew.total_seconds() if max_allowed < abs(skew_seconds): raise ClockSkew(skew_seconds) class B2Http: """ A wrapper for the requests module. Provides the operations needed to access B2, and handles retrying when the returned status is 503 Service Unavailable, 429 Too Many Requests, etc. The operations supported are: - post_json_return_json - post_content_return_json - get_content The methods that return JSON either return a Python dict or raise a subclass of B2Error. They can be used like this: .. code-block:: python try: response_dict = b2_http.post_json_return_json(url, headers, params) ... except B2Error as e: ... Please note that the timeout/retry system, including class-level variables, is not a part of the interface and is subject to change. """ CONNECTION_TIMEOUT = 3 + 6 + 12 + 24 + 1 # 4 standard tcp retransmissions + 1s latency TIMEOUT = 128 TIMEOUT_FOR_COPY = 1200 # 20 minutes as server-side copy can take time TIMEOUT_FOR_UPLOAD = 128 TRY_COUNT_DATA = 20 TRY_COUNT_DOWNLOAD = 20 TRY_COUNT_HEAD = 5 TRY_COUNT_OTHER = 5 def __init__(self, api_config: B2HttpApiConfig = DEFAULT_HTTP_API_CONFIG): """ Initialize with a reference to the requests module, which makes it easy to mock for testing. """ self.user_agent = self._get_user_agent(api_config.user_agent_append) self.session = api_config.http_session_factory() if not api_config.decode_content: self.session.adapters.clear() self.session.mount('', NotDecompressingHTTPAdapter()) self.callbacks = [] if api_config.install_clock_skew_hook: self.add_callback(ClockSkewHook()) def add_callback(self, callback): """ Add a callback that inherits from HttpCallback. :param callback: a callback to be added to a chain :type callback: callable """ self.callbacks.append(callback) def request( self, method: Literal['POST', 'GET', 'HEAD'], url: str, headers: dict[str, str], data: io.BytesIO | bytes | None = None, try_count: int = TRY_COUNT_DATA, params: dict[str, str] | None = None, *, stream: bool = False, _timeout: int | None = None, ) -> requests.Response: """ Use like this: .. code-block:: python try: response_dict = b2_http.request('POST', url, headers, data) ... except B2Error as e: ... :param method: uppercase HTTP method name :param url: a URL to call :param headers: headers to send. :param data: raw bytes or a file-like object to send :param try_count: a number of retries :param params: a dict that will be converted to query string for GET requests or additional metadata for POST requests :param stream: if True, the response will be streamed :param _timeout: a timeout for the request in seconds if not default :return: final response :raises: B2Error if the request fails """ method = method.upper() request_headers = {**headers, 'User-Agent': self.user_agent} def do_request(): # This may retry, so each time we need to rewind the data back to the beginning. if data is not None and not isinstance(data, bytes): data.seek(0) self._run_pre_request_hooks(method, url, request_headers) response = self.session.request( method, url, headers=request_headers, data=data, params=params if method == 'GET' else None, timeout=(self.CONNECTION_TIMEOUT, _timeout or self.TIMEOUT_FOR_UPLOAD), stream=stream, ) self._run_post_request_hooks(method, url, request_headers, response) return response return self._translate_and_retry(do_request, try_count, params) def request_content_return_json( self, method: Literal['POST', 'GET', 'HEAD'], url: str, headers: dict[str, str], data: io.BytesIO | bytes | None = None, try_count: int = TRY_COUNT_DATA, params: dict[str, str] | None = None, *, _timeout: int | None = None, ) -> JSON: """ Use like this: .. code-block:: python try: response_dict = b2_http.request_content_return_json('POST', url, headers, data) ... except B2Error as e: ... :param method: uppercase HTTP method name :param url: a URL to call :param headers: headers to send. :param data: raw bytes or a file-like object to send :return: decoded JSON """ response = self.request( method, url, headers={**headers, 'Accept': 'application/json'}, data=data, try_count=try_count, params=params, _timeout=_timeout, ) # Decode the JSON that came back. If we've gotten this far, # we know we have a status of 200 OK. In this case, the body # of the response is always JSON, so we don't need to handle # it being something else. try: return json.loads(response.content.decode('utf-8')) finally: response.close() def post_content_return_json( self, url: str, headers: dict[str, str], data: bytes | io.IOBase, try_count: int = TRY_COUNT_DATA, post_params: dict[str, str] | None = None, _timeout: int | None = None, ) -> JSON: """ Use like this: .. code-block:: python try: response_dict = b2_http.post_content_return_json(url, headers, data) ... except B2Error as e: ... :param str url: a URL to call :param dict headers: headers to send. :param data: a file-like object to send :return: a dict that is the decoded JSON """ try: return self.request_content_return_json( 'POST', url, headers, data, try_count, post_params, _timeout=_timeout ) except B2RequestTimeout: # this forces a token refresh, which is necessary if request is still alive # on the server but has terminated for some reason on the client. See #79 raise B2RequestTimeoutDuringUpload() def post_json_return_json(self, url, headers, params, try_count: int = TRY_COUNT_OTHER): """ Use like this: .. code-block:: python try: response_dict = b2_http.post_json_return_json(url, headers, params) ... except B2Error as e: ... :param str url: a URL to call :param dict headers: headers to send. :param dict params: a dict that will be converted to JSON :return: the decoded JSON document :rtype: dict """ # This is not just b2_copy_file or b2_copy_part, but it would not # be good to find out by analyzing the url. # In the future a more generic system between raw_api and b2http # to indicate the timeouts should be designed. timeout = self.TIMEOUT_FOR_COPY data = json.dumps(params).encode() return self.post_content_return_json( url, { **headers, 'Content-Type': 'application/json', }, data, try_count, params, _timeout=timeout, ) def get_content(self, url, headers, try_count: int = TRY_COUNT_DOWNLOAD): """ Fetches content from a URL. Use like this: .. code-block:: python try: with b2_http.get_content(url, headers) as response: for byte_data in response.iter_content(chunk_size=1024): ... except B2Error as e: ... The response object is only guarantee to have: - headers - iter_content() :param str url: a URL to call :param dict headers: headers to send :param int try_count: a number or retries :return: Context manager that returns an object that supports iter_content() """ response = self.request( 'GET', url, headers=headers, try_count=try_count, stream=True, _timeout=self.TIMEOUT ) return ResponseContextManager(response) def head_content( self, url: str, headers: dict[str, Any], try_count: int = TRY_COUNT_HEAD, ) -> requests.Response: """ Does a HEAD instead of a GET for the URL. The response's content is limited to the headers. Use like this: .. code-block:: python try: response_dict = b2_http.head_content(url, headers) ... except B2Error as e: ... The response object is only guaranteed to have: - headers :param str url: a URL to call :param dict headers: headers to send :param int try_count: a number or retries :return: HTTP response """ return self.request('HEAD', url, headers=headers, try_count=try_count) @classmethod def _get_user_agent(cls, user_agent_append): if user_agent_append: return f'{USER_AGENT} {user_agent_append}' return USER_AGENT def _run_pre_request_hooks(self, method, url, headers): for callback in self.callbacks: callback.pre_request(method, url, headers) def _run_post_request_hooks(self, method, url, headers, response): for callback in self.callbacks: callback.post_request(method, url, headers, response) @classmethod def _translate_errors(cls, fcn, post_params=None): """ Call the given function, turning any exception raised into the right kind of B2Error. :param dict post_params: request parameters """ response = None try: response = fcn() if response.status_code not in (200, 206): # Decode the error object returned by the service try: error = json.loads(response.content.decode('utf-8')) if response.content else {} if not isinstance(error, dict): raise ValueError('json error value is not a dict') except (json.JSONDecodeError, UnicodeDecodeError, ValueError): logger.error('failed to decode error response: %r', response.content) # When the user points to an S3 endpoint, he won't receive the JSON error # he expects. In that case, we can provide at least a hint of "what happened". # s3 url has the form of e.g. https://s3.us-west-000.backblazeb2.com if '://s3.' in response.url: raise PotentialS3EndpointPassedAsRealm(response.content) error = { 'message': response.content.decode('utf-8', errors='replace'), 'code': 'non_json_response', } extra_error_keys = error.keys() - ('code', 'status', 'message') if extra_error_keys: logger.debug( 'received error has extra (unsupported) keys: %s', extra_error_keys ) try: status = int(error.get('status', response.status_code)) if status != response.status_code: raise ValueError('status code is not equal to the one in the response') except (TypeError, ValueError) as exc: logger.warning( 'Inconsistent status codes returned by the server %r != %r; parsing exception: %r', error.get('status'), response.status_code, exc, ) status = response.status_code raise interpret_b2_error( status, str(error['code']) if 'code' in error else None, str(error['message']) if 'message' in error else None, response.headers, post_params, ) return response except B2Error: raise # pass through exceptions from just above except requests.ConnectionError as e0: e1 = e0.args[0] if isinstance(e1, requests.packages.urllib3.exceptions.MaxRetryError): msg = e1.args[0] if 'nodename nor servname provided, or not known' in msg: # Unknown host, or DNS failing. In the context of calling # B2, this means that something is down between here and # Backblaze, so we treat it like 503 Service Unavailable. raise UnknownHost() elif isinstance(e1, requests.packages.urllib3.exceptions.ProtocolError): e2 = e1.args[1] if isinstance(e2, socket.error): if len(e2.args) >= 2 and e2.args[1] == 'Broken pipe': # Broken pipes are usually caused by the service rejecting # an upload request for cause, so we use a 400 Bad Request # code. raise BrokenPipe() elif isinstance(e2, TimeoutError): raise B2RequestTimeout(str(e0)) raise B2ConnectionError(str(e0)) except requests.Timeout as e: raise B2RequestTimeout(str(e)) except Exception as e: text = repr(e) # This is a special case to handle when urllib3 doesn't translate # ECONNRESET into something that requests can turn into something # we understand. The SysCallError is from the optional library # pyOpenSsl, which we don't require, so we can't import it and # catch it explicitly. # # The text from one such error looks like this: SysCallError(104, 'ECONNRESET') if text.startswith('SysCallError'): if 'ECONNRESET' in text: raise ConnectionReset() logger.exception('_translate_errors has intercepted an unexpected exception') raise UnknownError(text) @classmethod def _translate_and_retry( cls, fcn: Callable, try_count: int, post_params: dict[str, Any] | None = None ): """ Try calling fcn try_count times, retrying only if the exception is a retryable B2Error. :param fcn: request function to call :param try_count: a number of retries :param post_params: request parameters """ # For all but the last try, catch the exception. wait_time = 1.0 max_wait_time = 64 for _ in range(try_count - 1): try: return cls._translate_errors(fcn, post_params) except B2Error as e: if not e.should_retry_http(): raise logger.debug(str(e), exc_info=True) if e.retry_after_seconds is not None: sleep_duration = e.retry_after_seconds sleep_reason = 'server asked us to' else: sleep_duration = wait_time sleep_reason = 'that is what the default exponential backoff is' logger.info( 'Pausing thread for %i seconds because %s', sleep_duration, sleep_reason, ) time.sleep(sleep_duration) # Set up wait time for the next iteration wait_time *= 1.5 if wait_time > max_wait_time: # avoid clients synchronizing and causing a wave # of requests when connectivity is restored wait_time = max_wait_time + random() # If the last try gets an exception, it will be raised. return cls._translate_errors(fcn, post_params) class NotDecompressingHTTPAdapter(HTTPAdapter): """ HTTP adapter that uses :class:`b2sdk._internal.requests.NotDecompressingResponse` instead of the default :code:`requests.Response` class. """ def build_response(self, req, resp): return NotDecompressingResponse.from_builtin_response(super().build_response(req, resp)) b2-sdk-python-2.8.0/b2sdk/_internal/bounded_queue_executor.py000066400000000000000000000044141474454370000242310ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/bounded_queue_executor.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import threading class BoundedQueueExecutor: """ Wrap a concurrent.futures.Executor and limits the number of requests that can be queued at once. Requests to submit() tasks block until there is room in the queue. The number of available slots in the queue is tracked with a semaphore that is acquired before queueing an action, and released when an action finishes. Counts the number of exceptions thrown by tasks, and makes them available from get_num_exceptions() after shutting down. """ def __init__(self, executor, queue_limit): """ :param executor: an executor to be wrapped :type executor: concurrent.futures.Executor :param queue_limit: a queue limit :type queue_limit: int """ self.executor = executor self.semaphore = threading.Semaphore(queue_limit) self.num_exceptions = 0 def submit(self, fcn, *args, **kwargs): """ Start execution of a callable with the given optional and positional arguments :param fcn: a callable object :type fcn: callable :return: a future object :rtype: concurrent.futures.Future """ # Wait until there is room in the queue. self.semaphore.acquire() # Wrap the action in a function that will release # the semaphore after it runs. def run_it(): try: return fcn(*args, **kwargs) except Exception: self.num_exceptions += 1 raise finally: self.semaphore.release() # Submit the wrapped action. return self.executor.submit(run_it) def shutdown(self): """ Shut an executor down. """ self.executor.shutdown() def get_num_exceptions(self): """ Return a number of exceptions. :rtype: int """ return self.num_exceptions b2-sdk-python-2.8.0/b2sdk/_internal/bucket.py000066400000000000000000002407721474454370000207550ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/bucket.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import datetime as dt import fnmatch import itertools import logging import pathlib from contextlib import suppress from typing import Iterable, Sequence from .encryption.setting import EncryptionSetting, EncryptionSettingFactory from .encryption.types import EncryptionMode from .exception import ( BucketIdNotFound, CopySourceTooBig, FileDeleted, FileNotHidden, FileNotPresent, FileOrBucketNotFound, UnexpectedCloudBehaviour, UnexpectedFileVersionAction, UnrecognizedBucketType, ) from .file_lock import ( UNKNOWN_BUCKET_RETENTION, BucketRetentionSetting, FileLockConfiguration, FileRetentionSetting, LegalHold, ) from .file_version import DownloadVersion, FileIdAndName, FileVersion from .filter import Filter, FilterMatcher from .http_constants import LIST_FILE_NAMES_MAX_LIMIT from .progress import AbstractProgressListener, DoNothingProgressListener from .raw_api import LifecycleRule, NotificationRule, NotificationRuleResponse from .replication.setting import ReplicationConfiguration, ReplicationConfigurationFactory from .transfer.emerge.executor import AUTO_CONTENT_TYPE from .transfer.emerge.unbound_write_intent import UnboundWriteIntentGenerator from .transfer.emerge.write_intent import WriteIntent from .transfer.inbound.downloaded_file import DownloadedFile from .transfer.outbound.copy_source import CopySource from .transfer.outbound.upload_source import UploadMode, UploadSourceBytes, UploadSourceLocalFile from .utils import ( B2TraceMeta, Sha1HexDigest, b2_url_encode, disable_trace, limit_trace_arguments, validate_b2_file_name, ) logger = logging.getLogger(__name__) class Bucket(metaclass=B2TraceMeta): """ Provide access to a bucket in B2: listing files, uploading and downloading. """ DEFAULT_CONTENT_TYPE = AUTO_CONTENT_TYPE def __init__( self, api, id_, name=None, type_=None, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, revision=None, bucket_dict=None, options_set=None, default_server_side_encryption: EncryptionSetting = EncryptionSetting( EncryptionMode.UNKNOWN ), default_retention: BucketRetentionSetting = UNKNOWN_BUCKET_RETENTION, is_file_lock_enabled: bool | None = None, replication: ReplicationConfiguration | None = None, ): """ :param b2sdk.v2.B2Api api: an API object :param str id_: a bucket id :param str name: a bucket name :param str type_: a bucket type :param dict bucket_info: an info to store with a bucket :param dict cors_rules: CORS rules to store with a bucket :param lifecycle_rules: lifecycle rules of the bucket :param int revision: a bucket revision number :param dict bucket_dict: a dictionary which contains bucket parameters :param set options_set: set of bucket options strings :param b2sdk.v2.EncryptionSetting default_server_side_encryption: default server side encryption settings :param b2sdk.v2.BucketRetentionSetting default_retention: default retention setting :param bool is_file_lock_enabled: whether file locking is enabled or not :param b2sdk.v2.ReplicationConfiguration replication: replication rules for the bucket """ self.api = api self.id_ = id_ self.name = name self.type_ = type_ self.bucket_info = bucket_info or {} self.cors_rules = cors_rules or [] self.lifecycle_rules = lifecycle_rules or [] self.revision = revision self.bucket_dict = bucket_dict or {} self.options_set = options_set or set() self.default_server_side_encryption = default_server_side_encryption self.default_retention = default_retention self.is_file_lock_enabled = is_file_lock_enabled self.replication = replication def _add_file_info_item(self, file_info: dict[str, str], name: str, value: str | None): if value is not None: if name in file_info and file_info[name] != value: logger.warning( 'Overwriting file info key %s with value %s (previous value %s)', name, value, file_info[name], ) file_info[name] = value def _merge_file_info_and_headers_params( self, file_info: dict | None, cache_control: str | None, expires: str | dt.datetime | None, content_disposition: str | None, content_encoding: str | None, content_language: str | None, ) -> dict | None: updated_file_info = {**(file_info or {})} if isinstance(expires, dt.datetime): expires = expires.astimezone(dt.timezone.utc) expires = dt.datetime.strftime(expires, '%a, %d %b %Y %H:%M:%S GMT') self._add_file_info_item(updated_file_info, 'b2-expires', expires) self._add_file_info_item(updated_file_info, 'b2-cache-control', cache_control) self._add_file_info_item(updated_file_info, 'b2-content-disposition', content_disposition) self._add_file_info_item(updated_file_info, 'b2-content-encoding', content_encoding) self._add_file_info_item(updated_file_info, 'b2-content-language', content_language) # If file_info was None and we didn't add anything, we want to return None if not updated_file_info: return file_info return updated_file_info def get_fresh_state(self) -> Bucket: """ Fetch all the information about this bucket and return a new bucket object. This method does NOT change the object it is called on. """ buckets_found = self.api.list_buckets(bucket_id=self.id_) if not buckets_found: raise BucketIdNotFound(self.id_) return buckets_found[0] def get_id(self) -> str: """ Return bucket ID. :rtype: str """ return self.id_ def set_info(self, new_bucket_info, if_revision_is=None) -> Bucket: """ Update bucket info. :param dict new_bucket_info: new bucket info dictionary :param int if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is* """ return self.update(bucket_info=new_bucket_info, if_revision_is=if_revision_is) def set_type(self, bucket_type) -> Bucket: """ Update bucket type. :param str bucket_type: a bucket type ("allPublic" or "allPrivate") """ return self.update(bucket_type=bucket_type) def update( self, bucket_type: str | None = None, bucket_info: dict | None = None, cors_rules: dict | None = None, lifecycle_rules: list[LifecycleRule] | None = None, if_revision_is: int | None = None, default_server_side_encryption: EncryptionSetting | None = None, default_retention: BucketRetentionSetting | None = None, replication: ReplicationConfiguration | None = None, is_file_lock_enabled: bool | None = None, ) -> Bucket: """ Update various bucket parameters. :param bucket_type: a bucket type, e.g. ``allPrivate`` or ``allPublic`` :param bucket_info: an info to store with a bucket :param cors_rules: CORS rules to store with a bucket :param lifecycle_rules: lifecycle rules to store with a bucket :param if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is* :param default_server_side_encryption: default server side encryption settings (``None`` if unknown) :param default_retention: bucket default retention setting :param replication: replication rules for the bucket :param bool is_file_lock_enabled: specifies whether bucket should get File Lock-enabled """ account_id = self.api.account_info.get_account_id() return self.api.BUCKET_FACTORY_CLASS.from_api_bucket_dict( self.api, self.api.session.update_bucket( account_id, self.id_, bucket_type=bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, replication=replication, is_file_lock_enabled=is_file_lock_enabled, ), ) def cancel_large_file(self, file_id): """ Cancel a large file transfer. :param str file_id: a file ID """ return self.api.cancel_large_file(file_id) def download_file_by_id( self, file_id: str, progress_listener: AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: EncryptionSetting | None = None, ) -> DownloadedFile: """ Download a file by ID. .. note:: download_file_by_id actually belongs in :py:class:`b2sdk.v2.B2Api`, not in :py:class:`b2sdk.v2.Bucket`; we just provide a convenient redirect here :param file_id: a file ID :param progress_listener: a progress listener object to use, or ``None`` to not track progress :param range_: two integer values, start and end offsets :param encryption: encryption settings (``None`` if unknown) """ return self.api.download_file_by_id( file_id, progress_listener, range_=range_, encryption=encryption, ) def download_file_by_name( self, file_name: str, progress_listener: AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: EncryptionSetting | None = None, ) -> DownloadedFile: """ Download a file by name. .. seealso:: :ref:`Synchronizer `, a *high-performance* utility that synchronizes a local folder with a Bucket. :param file_name: a file name :param progress_listener: a progress listener object to use, or ``None`` to not track progress :param range_: two integer values, start and end offsets :param encryption: encryption settings (``None`` if unknown) """ url = self.api.session.get_download_url_by_name(self.name, file_name) return self.api.services.download_manager.download_file_from_url( url, progress_listener, range_, encryption=encryption, ) def get_file_info_by_id(self, file_id: str) -> FileVersion: """ Gets a file version's by ID. :param str file_id: the id of the file who's info will be retrieved. :rtype: generator[b2sdk.v2.FileVersion] """ return self.api.get_file_info(file_id) def get_file_info_by_name(self, file_name: str) -> DownloadVersion: """ Gets a file's DownloadVersion by name. :param str file_name: the name of the file who's info will be retrieved. """ try: return self.api.download_version_factory.from_response_headers( self.api.session.get_file_info_by_name(self.name, file_name) ) except FileOrBucketNotFound: raise FileNotPresent(bucket_name=self.name, file_id_or_name=file_name) def get_download_authorization(self, file_name_prefix, valid_duration_in_seconds): """ Return an authorization token that is valid only for downloading files from the given bucket. :param str file_name_prefix: a file name prefix, only files that match it could be downloaded :param int valid_duration_in_seconds: a token is valid only during this amount of seconds """ response = self.api.session.get_download_authorization( self.id_, file_name_prefix, valid_duration_in_seconds ) return response['authorizationToken'] def list_parts(self, file_id, start_part_number=None, batch_size=None): """ Get a list of all parts that have been uploaded for a given file. :param str file_id: a file ID :param int start_part_number: the first part number to return. defaults to the first part. :param int batch_size: the number of parts to fetch at a time from the server """ return self.api.list_parts(file_id, start_part_number, batch_size) def list_file_versions( self, file_name: str, fetch_count: int | None = LIST_FILE_NAMES_MAX_LIMIT ) -> Iterable[FileVersion]: """ Lists all of the versions for a single file. :param file_name: the name of the file to list. :param fetch_count: how many entries to list per API call or ``None`` to use the default. Acceptable values: 1 - 10000 :rtype: generator[b2sdk.v2.FileVersion] """ if fetch_count is not None and fetch_count <= 0: # fetch_count equal to 0 means "use API default", which we don't want to support here raise ValueError('unsupported fetch_count value') start_file_name = file_name start_file_id = None session = self.api.session while 1: response = session.list_file_versions( self.id_, start_file_name, start_file_id, fetch_count, file_name ) for entry in response['files']: file_version = self.api.file_version_factory.from_api_response(entry) if file_version.file_name != file_name: # All versions for the requested file name have been listed. return yield file_version start_file_name = response['nextFileName'] start_file_id = response['nextFileId'] if start_file_name is None: return def ls( self, path: str = '', latest_only: bool = True, recursive: bool = False, fetch_count: int | None = LIST_FILE_NAMES_MAX_LIMIT, with_wildcard: bool = False, filters: Sequence[Filter] = (), ) -> Iterable[tuple[FileVersion, str]]: """ Pretend that folders exist and yields the information about the files in a folder. B2 has a flat namespace for the files in a bucket, but there is a convention of using "/" as if there were folders. This method searches through the flat namespace to find the files and "folders" that live within a given folder. When the `recursive` flag is set, lists all of the files in the given folder, and all of its sub-folders. :param path: Path to list. To reduce the number of API calls, if path points to a folder, it should end with "/". Must not start with "/". Empty string means top-level folder. :param latest_only: when ``False`` returns info about all versions of a file, when ``True``, just returns info about the most recent versions :param recursive: if ``True``, list folders recursively :param fetch_count: how many entries to list per API call or ``None`` to use the default. Acceptable values: 1 - 10000 :param with_wildcard: Accepts "*", "?", "[]" and "[!]" in folder_to_list, similarly to what shell does. As of 1.19.0 it can only be enabled when recursive is also enabled. Also, in this mode, folder_to_list is considered to be a filename or a pattern. :param filters: list of filters to apply to the files returned by the server. :rtype: generator[tuple[b2sdk.v2.FileVersion, str]] :returns: generator of (file_version, folder_name) tuples .. note:: In case of `recursive=True`, folder_name is not returned. """ # Ensure that recursive is enabled when with_wildcard is enabled. if with_wildcard and not recursive: raise ValueError('with_wildcard requires recursive to be turned on as well') # check if path points to an object instead of a folder if path and not with_wildcard and not path.endswith('/'): file_versions = self.list_file_versions(path, 1 if latest_only else fetch_count) if latest_only: file_versions = itertools.islice(file_versions, 1) path_pointed_to_file = False for file_version in file_versions: path_pointed_to_file = True if not latest_only or file_version.action == 'upload': yield file_version, None if path_pointed_to_file: return folder_to_list = path # Every file returned must have a name that starts with the # folder name and a "/". prefix = folder_to_list # In case of wildcards, we don't assume that this is folder that we're searching through. # It could be an exact file, e.g. 'a/b.txt' that we're trying to locate. if prefix != '' and not prefix.endswith('/') and not with_wildcard: prefix += '/' # If we're running with wildcard-matching, we could get # a different prefix from it. We search for the first # occurrence of the special characters and fetch # parent path from that place. # Examples: # 'b/c/*.txt' –> 'b/c/' # '*.txt' –> '' # 'a/*/result.[ct]sv' –> 'a/' if with_wildcard: for wildcard_character in '*?[': try: starter_index = folder_to_list.index(wildcard_character) except ValueError: continue # +1 to include the starter character. Using posix path to # ensure consistent behaviour on Windows (e.g. case sensitivity). path = pathlib.PurePosixPath(folder_to_list[: starter_index + 1]) parent_path = str(path.parent) # Path considers dot to be the empty path. # There's no shorter path than that. if parent_path == '.': prefix = '' break # We could receive paths in different stage, e.g. 'a/*/result.[ct]sv' has two # possible parent paths: 'a/' and 'a/*/', with the first one being the correct one if len(parent_path) < len(prefix): prefix = parent_path # Loop until all files in the named directory have been listed. # The starting point of the first list_file_names request is the # prefix we're looking for. The prefix ends with '/', which is # now allowed for file names, so no file name will match exactly, # but the first one after that point is the first file in that # "folder". If the first search doesn't produce enough results, # then we keep calling list_file_names until we get all of the # names in this "folder". filter_matcher = FilterMatcher(filters) current_dir = None start_file_name = prefix start_file_id = None session = self.api.session while True: if latest_only: response = session.list_file_names(self.id_, start_file_name, fetch_count, prefix) else: response = session.list_file_versions( self.id_, start_file_name, start_file_id, fetch_count, prefix ) for entry in response['files']: file_version = self.api.file_version_factory.from_api_response(entry) if not file_version.file_name.startswith(prefix): # We're past the files we care about return if with_wildcard and not fnmatch.fnmatchcase( file_version.file_name, folder_to_list ): # File doesn't match our wildcard rules continue if not filter_matcher.match(file_version.file_name): continue after_prefix = file_version.file_name[len(prefix) :] # In case of wildcards, we don't care about folders at all, and it's recursive by default. if '/' not in after_prefix or recursive: # This is not a folder, so we'll print it out and # continue on. yield file_version, None current_dir = None else: # This is a folder. If it's different than the folder # we're already in, then we can print it. This check # is needed, because all of the files in the folder # will be in the list. folder_with_slash = after_prefix.split('/')[0] + '/' if folder_with_slash != current_dir: folder_name = prefix + folder_with_slash yield file_version, folder_name current_dir = folder_with_slash if response['nextFileName'] is None: # The response says there are no more files in the bucket, # so we can stop. return # Now we need to set up the next search. The response from # B2 has the starting point to continue with the next file, # but if we're in the middle of a "folder", we can skip ahead # to the end of the folder. The character after '/' is '0', # so we'll replace the '/' with a '0' and start there. # # When recursive is True, current_dir is always None. if current_dir is None: start_file_name = response.get('nextFileName') start_file_id = response.get('nextFileId') else: start_file_name = max( response['nextFileName'], prefix + current_dir[:-1] + '0', ) def list_unfinished_large_files(self, start_file_id=None, batch_size=None, prefix=None): """ A generator that yields an :py:class:`b2sdk.v2.UnfinishedLargeFile` for each unfinished large file in the bucket, starting at the given file, filtering by prefix. :param str,None start_file_id: a file ID to start from or None to start from the beginning :param int,None batch_size: max file count :param str,None prefix: file name prefix filter :rtype: generator[b2sdk.v2.UnfinishedLargeFile] """ return self.api.services.large_file.list_unfinished_large_files( self.id_, start_file_id=start_file_id, batch_size=batch_size, prefix=prefix, ) @limit_trace_arguments(skip=('data_bytes',)) def upload_bytes( self, data_bytes, file_name, content_type: str | None = None, file_info: dict | None = None, progress_listener=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, large_file_sha1: Sha1HexDigest | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, expires: str | dt.datetime | None = None, content_disposition: str | None = None, content_encoding: str | None = None, content_language: str | None = None, ) -> FileVersion: """ Upload bytes in memory to a B2 file. .. note: ``custom_upload_timestamp`` is disabled by default - please talk to customer support to enable it on your account (if you really need it) :param bytes data_bytes: a byte array to upload :param str file_name: a file name to upload bytes to :param str,None content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param dict,None file_info: a file info to store with the file or ``None`` to not store anything :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not track progress :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param Sha1HexDigest,None large_file_sha1: SHA-1 hash of the result file or ``None`` if unknown :param int,None custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch :param str,None cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. :param str,datetime.datetime,None expires: an optional cache expiration setting. If this argument is a string, its syntax must be based on the section 14.21 of RFC 2616. Example string value: 'Thu, 01 Dec 2050 16:00:00 GMT'. If this argument is a datetime, it will be converted to a string in the same format. :param str,None content_disposition: an optional content disposition setting. Syntax based on the section 19.5.1 of RFC 2616. Example string value: 'attachment; filename="fname.ext"'. :param str,None content_encoding: an optional content encoding setting.Syntax based on the section 14.11 of RFC 2616. Example string value: 'gzip'. :param str,None content_language: an optional content language setting. Syntax based on the section 14.12 of RFC 2616. Example string value: 'mi, en_US'. :rtype: b2sdk.v2.FileVersion """ upload_source = UploadSourceBytes(data_bytes) return self.upload( upload_source, file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, large_file_sha1=large_file_sha1, custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expires=expires, content_disposition=content_disposition, content_encoding=content_encoding, content_language=content_language, ) def upload_local_file( self, local_file, file_name, content_type: str | None = None, file_info: dict | None = None, sha1_sum: str | None = None, min_part_size: int | None = None, progress_listener=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, upload_mode: UploadMode = UploadMode.FULL, custom_upload_timestamp: int | None = None, cache_control: str | None = None, expires: str | dt.datetime | None = None, content_disposition: str | None = None, content_encoding: str | None = None, content_language: str | None = None, ): """ Upload a file on local disk to a B2 file. .. note: ``custom_upload_timestamp`` is disabled by default - please talk to customer support to enable it on your account (if you really need it) .. seealso:: :ref:`Synchronizer `, a *high-performance* utility that synchronizes a local folder with a :term:`bucket`. :param str local_file: a path to a file on local disk :param str file_name: a file name of the new B2 file :param content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param file_info: a file info to store with the file or ``None`` to not store anything :param sha1_sum: file SHA1 hash or ``None`` to compute it automatically :param min_part_size: lower limit of part size for the transfer planner, in bytes :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param b2sdk.v2.UploadMode upload_mode: desired upload mode :param int,None custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch :param str,None cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. :param str,datetime.datetime,None expires: an optional cache expiration setting. If this argument is a string, its syntax must be based on the section 14.21 of RFC 2616. Example string value: 'Thu, 01 Dec 2050 16:00:00 GMT'. If this argument is a datetime, it will be converted to a string in the same format. :param str,None content_disposition: an optional content disposition setting. Syntax based on the section 19.5.1 of RFC 2616. Example string value: 'attachment; filename="fname.ext"'. :param str,None content_encoding: an optional content encoding setting.Syntax based on the section 14.11 of RFC 2616. Example string value: 'gzip'. :param str,None content_language: an optional content language setting. Syntax based on the section 14.12 of RFC 2616. Example string value: 'mi, en_US'. :rtype: b2sdk.v2.FileVersion """ upload_source = UploadSourceLocalFile(local_path=local_file, content_sha1=sha1_sum) sources = [upload_source] large_file_sha1 = sha1_sum if upload_mode == UploadMode.INCREMENTAL: with suppress(FileNotPresent): existing_file_info = self.get_file_info_by_name(file_name) sources = upload_source.get_incremental_sources( existing_file_info, self.api.session.account_info.get_absolute_minimum_part_size(), ) if len(sources) > 1 and not large_file_sha1: # the upload will be incremental, but the SHA1 sum is unknown, calculate it now large_file_sha1 = upload_source.get_content_sha1() file_info = self._merge_file_info_and_headers_params( file_info=file_info, cache_control=cache_control, expires=expires, content_disposition=content_disposition, content_encoding=content_encoding, content_language=content_language, ) return self.concatenate( sources, file_name, content_type=content_type, file_info=file_info, min_part_size=min_part_size, progress_listener=progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, large_file_sha1=large_file_sha1, custom_upload_timestamp=custom_upload_timestamp, ) def upload_unbound_stream( self, read_only_object, file_name: str, content_type: str = None, file_info: dict[str, str] | None = None, progress_listener: AbstractProgressListener | None = None, recommended_upload_part_size: int | None = None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, min_part_size: int | None = None, max_part_size: int | None = None, large_file_sha1: Sha1HexDigest | None = None, buffers_count: int = 2, buffer_size: int | None = None, read_size: int = 8192, unused_buffer_timeout_seconds: float = 3600.0, custom_upload_timestamp: int | None = None, cache_control: str | None = None, expires: str | dt.datetime | None = None, content_disposition: str | None = None, content_encoding: str | None = None, content_language: str | None = None, ): """ Upload an unbound file-like read-only object to a B2 file. It is assumed that this object is streamed like stdin or socket, and the size is not known up front. It is up to caller to ensure that this object is open and available through the whole streaming process. If stdin is to be passed, consider opening it in binary mode, if possible on the platform: .. code-block:: python with open(sys.stdin.fileno(), mode='rb', buffering=min_part_size, closefd=False) as source: bucket.upload_unbound_stream(source, 'target-file') For platforms without file descriptors, one can use the following: .. code-block:: python bucket.upload_unbound_stream(sys.stdin.buffer, 'target-file') but note that buffering in this case depends on the interpreter mode. ``min_part_size``, ``recommended_upload_part_size`` and ``max_part_size`` should all be greater than ``account_info.get_absolute_minimum_part_size()``. ``buffers_count`` describes a desired number of buffers that are to be used. Minimal amount is 2. to determine the method of uploading this stream (if there's only a single buffer we send it as a normal file, if there are at least two – as a large file). Number of buffers determines the amount of memory used by the streaming process and the amount of data that can be pulled from ``read_only_object`` while also uploading it. Providing more buffers allows for higher upload parallelization. While only one buffer can be filled with data at once, all others are used to send the data in parallel (limited only by the number of parallel threads). Buffer size can be controlled by ``buffer_size`` parameter. If left unset, it will default to a value of ``recommended_upload_part_size``. Note that in the current implementation buffers are (almost) directly sent to B2, thus whatever is picked as the ``buffer_size`` will also become the size of the part when uploading a large file in this manner. In rare cases, namely when the whole buffer was sent, but there was an error during sending of last bytes and a retry was issued, additional buffer (above the aforementioned limit) will be temporarily allocated. .. note: ``custom_upload_timestamp`` is disabled by default - please talk to customer support to enable it on your account (if you really need it) :param read_only_object: any object containing a ``read`` method accepting size of the read :param file_name: a file name of the new B2 file :param content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param file_info: a file info to store with the file or ``None`` to not store anything :param progress_listener: a progress listener object to use, or ``None`` to not report progress :param encryption: encryption settings (``None`` if unknown) :param file_retention: file retention setting :param legal_hold: legal hold setting :param recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically :param min_part_size: lower limit of part size for the transfer planner, in bytes :param max_part_size: upper limit of part size for the transfer planner, in bytes :param large_file_sha1: SHA-1 hash of the result file or ``None`` if unknown :param buffers_count: desired number of buffers allocated, cannot be smaller than 2 :param buffer_size: size of a single buffer that we pull data to or upload data to B2. If ``None``, value of ``recommended_upload_part_size`` is used. If that also is ``None``, it will be determined automatically as "recommended upload size". :param read_size: size of a single read operation performed on the ``read_only_object`` :param unused_buffer_timeout_seconds: amount of time that a buffer can be idle before returning error :param int,None custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch :param str,None cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. :param str,datetime.datetime,None expires: an optional cache expiration setting. If this argument is a string, its syntax must be based on the section 14.21 of RFC 2616. Example string value: 'Thu, 01 Dec 2050 16:00:00 GMT'. If this argument is a datetime, it will be converted to a string in the same format. :param str,None content_disposition: an optional content disposition setting. Syntax based on the section 19.5.1 of RFC 2616. Example string value: 'attachment; filename="fname.ext"'. :param str,None content_encoding: an optional content encoding setting.Syntax based on the section 14.11 of RFC 2616. Example string value: 'gzip'. :param str,None content_language: an optional content language setting. Syntax based on the section 14.12 of RFC 2616. Example string value: 'mi, en_US'. :rtype: b2sdk.v2.FileVersion """ if buffers_count <= 1: raise ValueError('buffers_count has to be at least 2') if read_size <= 0: raise ValueError('read_size has to be a positive integer') if unused_buffer_timeout_seconds <= 0.0: raise ValueError('unused_buffer_timeout_seconds has to be a positive float') buffer_size = buffer_size or recommended_upload_part_size if buffer_size is None: planner = self.api.services.emerger.get_emerge_planner() buffer_size = planner.recommended_upload_part_size file_info = self._merge_file_info_and_headers_params( file_info=file_info, cache_control=cache_control, expires=expires, content_disposition=content_disposition, content_encoding=content_encoding, content_language=content_language, ) return self._create_file( self.api.services.emerger.emerge_unbound, UnboundWriteIntentGenerator( read_only_object, buffer_size, read_size=read_size, queue_size=buffers_count, queue_timeout_seconds=unused_buffer_timeout_seconds, ).iterator(), file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, min_part_size=min_part_size, recommended_upload_part_size=recommended_upload_part_size, max_part_size=max_part_size, # This is a parameter for EmergeExecutor.execute_emerge_plan telling # how many buffers in parallel can be handled at once. We ensure that one buffer # is always downloading data from the stream while others are being uploaded. max_queue_size=buffers_count - 1, large_file_sha1=large_file_sha1, custom_upload_timestamp=custom_upload_timestamp, ) def upload( self, upload_source, file_name, content_type: str | None = None, file_info=None, min_part_size: int | None = None, progress_listener=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, large_file_sha1: Sha1HexDigest | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, expires: str | dt.datetime | None = None, content_disposition: str | None = None, content_encoding: str | None = None, content_language: str | None = None, ): """ Upload a file to B2, retrying as needed. The source of the upload is an UploadSource object that can be used to open (and re-open) the file. The result of opening should be a binary file whose read() method returns bytes. The function `opener` should return a file-like object, and it must be possible to call it more than once in case the upload is retried. .. note: ``custom_upload_timestamp`` is disabled by default - please talk to customer support to enable it on your account (if you really need it) :param b2sdk.v2.AbstractUploadSource upload_source: an object that opens the source of the upload :param str file_name: the file name of the new B2 file :param str,None content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param dict,None file_info: a file info to store with the file or ``None`` to not store anything :param min_part_size: lower limit of part size for the transfer planner, in bytes :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param Sha1HexDigest,None large_file_sha1: SHA-1 hash of the result file or ``None`` if unknown :param int,None custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch :param str,None cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. :param str,datetime.datetime,None expires: an optional cache expiration setting. If this argument is a string, its syntax must be based on the section 14.21 of RFC 2616. Example string value: 'Thu, 01 Dec 2050 16:00:00 GMT'. If this argument is a datetime, it will be converted to a string in the same format. :param str,None content_disposition: an optional content disposition setting. Syntax based on the section 19.5.1 of RFC 2616. Example string value: 'attachment; filename="fname.ext"'. :param str,None content_encoding: an optional content encoding setting.Syntax based on the section 14.11 of RFC 2616. Example string value: 'gzip'. :param str,None content_language: an optional content language setting. Syntax based on the section 14.12 of RFC 2616. Example string value: 'mi, en_US'. :rtype: b2sdk.v2.FileVersion """ return self.create_file( [WriteIntent(upload_source)], file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, # FIXME: Bucket.upload documents wrong logic recommended_upload_part_size=min_part_size, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, large_file_sha1=large_file_sha1, custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expires=expires, content_disposition=content_disposition, content_encoding=content_encoding, content_language=content_language, ) def create_file( self, write_intents, file_name, content_type: str | None = None, file_info=None, progress_listener=None, recommended_upload_part_size=None, continue_large_file_id=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, min_part_size: int | None = None, max_part_size: int | None = None, large_file_sha1=None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, expires: str | dt.datetime | None = None, content_disposition: str | None = None, content_encoding: str | None = None, content_language: str | None = None, ): """ Creates a new file in this bucket using an iterable (list, tuple etc) of remote or local sources. Source ranges can overlap and remote sources will be prioritized over local sources (when possible). For more information and usage examples please see :ref:`Advanced usage patterns `. .. note: ``custom_upload_timestamp`` is disabled by default - please talk to customer support to enable it on your account (if you really need it) :param list[b2sdk.v2.WriteIntent] write_intents: list of write intents (remote or local sources) :param str file_name: file name of the new file :param str,None content_type: content_type for the new file, if ``None`` content_type would be automatically determined or it may be copied if it resolves as single part remote source copy :param dict,None file_info: file_info for the new file, if ``None`` it will be set to empty dict or it may be copied if it resolves as single part remote source copy :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param int,None recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, ``None`` for automatic search for this id :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param min_part_size: lower limit of part size for the transfer planner, in bytes :param max_part_size: upper limit of part size for the transfer planner, in bytes :param Sha1HexDigest,None large_file_sha1: SHA-1 hash of the result file or ``None`` if unknown :param int,None custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch :param str,None cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. :param str,datetime.datetime,None expires: an optional cache expiration setting. If this argument is a string, its syntax must be based on the section 14.21 of RFC 2616. Example string value: 'Thu, 01 Dec 2050 16:00:00 GMT'. If this argument is a datetime, it will be converted to a string in the same format. :param str,None content_disposition: an optional content disposition setting. Syntax based on the section 19.5.1 of RFC 2616. Example string value: 'attachment; filename="fname.ext"'. :param str,None content_encoding: an optional content encoding setting.Syntax based on the section 14.11 of RFC 2616. Example string value: 'gzip'. :param str,None content_language: an optional content language setting. Syntax based on the section 14.12 of RFC 2616. Example string value: 'mi, en_US'. """ return self._create_file( self.api.services.emerger.emerge, write_intents, file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, continue_large_file_id=continue_large_file_id, recommended_upload_part_size=recommended_upload_part_size, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, min_part_size=min_part_size, max_part_size=max_part_size, large_file_sha1=large_file_sha1, custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expires=expires, content_disposition=content_disposition, content_encoding=content_encoding, content_language=content_language, ) def create_file_stream( self, write_intents_iterator, file_name, content_type=None, file_info=None, progress_listener=None, recommended_upload_part_size=None, continue_large_file_id=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, min_part_size: int | None = None, max_part_size: int | None = None, large_file_sha1=None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, expires: str | dt.datetime | None = None, content_disposition: str | None = None, content_encoding: str | None = None, content_language: str | None = None, ): """ Creates a new file in this bucket using a stream of multiple remote or local sources. Source ranges can overlap and remote sources will be prioritized over local sources (when possible). For more information and usage examples please see :ref:`Advanced usage patterns `. .. note: ``custom_upload_timestamp`` is disabled by default - please talk to customer support to enable it on your account (if you really need it) :param iterator[b2sdk.v2.WriteIntent] write_intents_iterator: iterator of write intents which are sorted ascending by ``destination_offset`` :param str file_name: file name of the new file :param str,None content_type: content_type for the new file, if ``None`` content_type would be automatically determined or it may be copied if it resolves as single part remote source copy :param dict,None file_info: file_info for the new file, if ``None`` it will be set to empty dict or it may be copied if it resolves as single part remote source copy :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param int,None recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, if ``None`` in multipart case it would always start a new large file :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param min_part_size: lower limit of part size for the transfer planner, in bytes :param max_part_size: upper limit of part size for the transfer planner, in bytes :param Sha1HexDigest,None large_file_sha1: SHA-1 hash of the result file or ``None`` if unknown :param int,None custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch :param str,None cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. :param str,datetime.datetime,None expires: an optional cache expiration setting. If this argument is a string, its syntax must be based on the section 14.21 of RFC 2616. Example string value: 'Thu, 01 Dec 2050 16:00:00 GMT'. If this argument is a datetime, it will be converted to a string in the same format. :param str,None content_disposition: an optional content disposition setting. Syntax based on the section 19.5.1 of RFC 2616. Example string value: 'attachment; filename="fname.ext"'. :param str,None content_encoding: an optional content encoding setting.Syntax based on the section 14.11 of RFC 2616. Example string value: 'gzip'. :param str,None content_language: an optional content language setting. Syntax based on the section 14.12 of RFC 2616. Example string value: 'mi, en_US'. """ return self._create_file( self.api.services.emerger.emerge_stream, write_intents_iterator, file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, continue_large_file_id=continue_large_file_id, recommended_upload_part_size=recommended_upload_part_size, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, min_part_size=min_part_size, max_part_size=max_part_size, large_file_sha1=large_file_sha1, custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expires=expires, content_disposition=content_disposition, content_encoding=content_encoding, content_language=content_language, ) def _create_file( self, emerger_method, write_intents_iterable, file_name, content_type: str | None = None, file_info=None, progress_listener=None, recommended_upload_part_size=None, continue_large_file_id=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, min_part_size: int | None = None, max_part_size: int | None = None, large_file_sha1=None, cache_control: str | None = None, expires: str | dt.datetime | None = None, content_disposition: str | None = None, content_encoding: str | None = None, content_language: str | None = None, **kwargs, ): validate_b2_file_name(file_name) progress_listener = progress_listener or DoNothingProgressListener() file_info = self._merge_file_info_and_headers_params( file_info=file_info, cache_control=cache_control, expires=expires, content_disposition=content_disposition, content_encoding=content_encoding, content_language=content_language, ) return emerger_method( self.id_, write_intents_iterable, file_name, content_type, file_info, progress_listener, recommended_upload_part_size=recommended_upload_part_size, continue_large_file_id=continue_large_file_id, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, min_part_size=min_part_size, max_part_size=max_part_size, large_file_sha1=large_file_sha1, **kwargs, ) def concatenate( self, outbound_sources, file_name, content_type=None, file_info=None, progress_listener=None, recommended_upload_part_size=None, continue_large_file_id=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, min_part_size: int | None = None, max_part_size: int | None = None, large_file_sha1=None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, expires: str | dt.datetime | None = None, content_disposition: str | None = None, content_encoding: str | None = None, content_language: str | None = None, ): """ Creates a new file in this bucket by concatenating multiple remote or local sources. .. note: ``custom_upload_timestamp`` is disabled by default - please talk to customer support to enable it on your account (if you really need it) :param list[b2sdk.v2.OutboundTransferSource] outbound_sources: list of outbound sources (remote or local) :param str file_name: file name of the new file :param str,None content_type: content_type for the new file, if ``None`` content_type would be automatically determined from file name or it may be copied if it resolves as single part remote source copy :param dict,None file_info: file_info for the new file, if ``None`` it will be set to empty dict or it may be copied if it resolves as single part remote source copy :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param int,None recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, ``None`` for automatic search for this id :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param min_part_size: lower limit of part size for the transfer planner, in bytes :param max_part_size: upper limit of part size for the transfer planner, in bytes :param Sha1HexDigest,None large_file_sha1: SHA-1 hash of the result file or ``None`` if unknown :param int,None custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch :param str,None cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. :param str,datetime.datetime,None expires: an optional cache expiration setting. If this argument is a string, its syntax must be based on the section 14.21 of RFC 2616. Example string value: 'Thu, 01 Dec 2050 16:00:00 GMT'. If this argument is a datetime, it will be converted to a string in the same format. :param str,None content_disposition: an optional content disposition setting. Syntax based on the section 19.5.1 of RFC 2616. Example string value: 'attachment; filename="fname.ext"'. :param str,None content_encoding: an optional content encoding setting.Syntax based on the section 14.11 of RFC 2616. Example string value: 'gzip'. :param str,None content_language: an optional content language setting. Syntax based on the section 14.12 of RFC 2616. Example string value: 'mi, en_US'. """ return self.create_file( list(WriteIntent.wrap_sources_iterator(outbound_sources)), file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, recommended_upload_part_size=recommended_upload_part_size, continue_large_file_id=continue_large_file_id, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, min_part_size=min_part_size, max_part_size=max_part_size, large_file_sha1=large_file_sha1, custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expires=expires, content_disposition=content_disposition, content_encoding=content_encoding, content_language=content_language, ) def concatenate_stream( self, outbound_sources_iterator, file_name, content_type=None, file_info=None, progress_listener=None, recommended_upload_part_size=None, continue_large_file_id=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, large_file_sha1: Sha1HexDigest | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, expires: str | dt.datetime | None = None, content_disposition: str | None = None, content_encoding: str | None = None, content_language: str | None = None, ): """ Creates a new file in this bucket by concatenating stream of multiple remote or local sources. :param iterator[b2sdk.v2.OutboundTransferSource] outbound_sources_iterator: iterator of outbound sources :param str file_name: file name of the new file :param str,None content_type: content_type for the new file, if ``None`` content_type would be automatically determined or it may be copied if it resolves as single part remote source copy :param dict,None file_info: file_info for the new file, if ``None`` it will be set to empty dict or it may be copied if it resolves as single part remote source copy :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param int,None recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, if ``None`` in multipart case it would always start a new large file :param b2sdk.v2.EncryptionSetting encryption: encryption setting (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param bool legal_hold: legal hold setting :param Sha1HexDigest,None large_file_sha1: SHA-1 hash of the result file or ``None`` if unknown :param int,None custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch :param str,None cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. :param str,datetime.datetime,None expires: an optional cache expiration setting. If this argument is a string, its syntax must be based on the section 14.21 of RFC 2616. Example string value: 'Thu, 01 Dec 2050 16:00:00 GMT'. If this argument is a datetime, it will be converted to a string in the same format. :param str,None content_disposition: an optional content disposition setting. Syntax based on the section 19.5.1 of RFC 2616. Example string value: 'attachment; filename="fname.ext"'. :param str,None content_encoding: an optional content encoding setting.Syntax based on the section 14.11 of RFC 2616. Example string value: 'gzip'. :param str,None content_language: an optional content language setting. Syntax based on the section 14.12 of RFC 2616. Example string value: 'mi, en_US'. """ return self.create_file_stream( WriteIntent.wrap_sources_iterator(outbound_sources_iterator), file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, recommended_upload_part_size=recommended_upload_part_size, continue_large_file_id=continue_large_file_id, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, large_file_sha1=large_file_sha1, custom_upload_timestamp=custom_upload_timestamp, cache_control=cache_control, expires=expires, content_disposition=content_disposition, content_encoding=content_encoding, content_language=content_language, ) def get_download_url(self, filename): """ Get file download URL. :param str filename: a file name :rtype: str """ return f'{self.api.account_info.get_download_url()}/file/{b2_url_encode(self.name)}/{b2_url_encode(filename)}' def hide_file(self, file_name): """ Hide a file. :param str file_name: a file name :rtype: b2sdk.v2.FileVersion """ response = self.api.session.hide_file(self.id_, file_name) return self.api.file_version_factory.from_api_response(response) def unhide_file(self, file_name: str, bypass_governance: bool = False) -> FileIdAndName: """ Unhide a file by deleting the "hide marker". """ # get the latest file version file_versions = self.list_file_versions(file_name=file_name, fetch_count=1) latest_file_version = next(file_versions, None) if latest_file_version is None: raise FileNotPresent(bucket_name=self.name, file_id_or_name=file_name) action = latest_file_version.action if action == 'upload': raise FileNotHidden(file_name) elif action == 'delete': raise FileDeleted(file_name) elif action != 'hide': raise UnexpectedFileVersionAction(action) return self.delete_file_version(latest_file_version.id_, file_name, bypass_governance) def copy( self, file_id, new_file_name, content_type=None, file_info=None, offset=0, length=None, progress_listener=None, destination_encryption: EncryptionSetting | None = None, source_encryption: EncryptionSetting | None = None, source_file_info: dict | None = None, source_content_type: str | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, cache_control: str | None = None, min_part_size: int | None = None, max_part_size: int | None = None, expires: str | dt.datetime | None = None, content_disposition: str | None = None, content_encoding: str | None = None, content_language: str | None = None, ) -> FileVersion: """ Creates a new file in this bucket by (server-side) copying from an existing file. :param str file_id: file ID of existing file to copy from :param str new_file_name: file name of the new file :param str,None content_type: content_type for the new file, if ``None`` and ``b2_copy_file`` will be used content_type will be copied from source file - otherwise content_type would be automatically determined :param dict,None file_info: file_info for the new file, if ``None`` will and ``b2_copy_file`` will be used file_info will be copied from source file - otherwise it will be set to empty dict :param int offset: offset of existing file that copy should start from :param int,None length: number of bytes to copy, if ``None`` then ``offset`` have to be ``0`` and it will use ``b2_copy_file`` without ``range`` parameter so it may fail if file is too large. For large files length have to be specified to use ``b2_copy_part`` instead. :param b2sdk.v2.AbstractProgressListener,None progress_listener: a progress listener object to use for multipart copy, or ``None`` to not report progress :param b2sdk.v2.EncryptionSetting destination_encryption: encryption settings for the destination (``None`` if unknown) :param b2sdk.v2.EncryptionSetting source_encryption: encryption settings for the source (``None`` if unknown) :param dict,None source_file_info: source file's file_info dict, useful when copying files with SSE-C :param str,None source_content_type: source file's content type, useful when copying files with SSE-C :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting for the new file. :param bool legal_hold: legal hold setting for the new file. :param str,None cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. :param min_part_size: lower limit of part size for the transfer planner, in bytes :param max_part_size: upper limit of part size for the transfer planner, in bytes :param str,datetime.datetime,None expires: an optional cache expiration setting. If this argument is a string, its syntax must be based on the section 14.21 of RFC 2616. Example string value: 'Thu, 01 Dec 2050 16:00:00 GMT'. If this argument is a datetime, it will be converted to a string in the same format. :param str,None content_disposition: an optional content disposition setting. Syntax based on the section 19.5.1 of RFC 2616. Example string value: 'attachment; filename="fname.ext"'. :param str,None content_encoding: an optional content encoding setting.Syntax based on the section 14.11 of RFC 2616. Example string value: 'gzip'. :param str,None content_language: an optional content language setting. Syntax based on the section 14.12 of RFC 2616. Example string value: 'mi, en_US'. """ copy_source = CopySource( file_id, offset=offset, length=length, encryption=source_encryption, source_file_info=source_file_info, source_content_type=source_content_type, ) if not length: # TODO: it feels like this should be checked on lower level - eg. RawApi validate_b2_file_name(new_file_name) try: progress_listener = progress_listener or DoNothingProgressListener() file_info = self._merge_file_info_and_headers_params( file_info=file_info, cache_control=cache_control, expires=expires, content_disposition=content_disposition, content_encoding=content_encoding, content_language=content_language, ) return self.api.services.copy_manager.copy_file( copy_source, new_file_name, content_type=content_type, file_info=file_info, destination_bucket_id=self.id_, progress_listener=progress_listener, destination_encryption=destination_encryption, source_encryption=source_encryption, file_retention=file_retention, legal_hold=legal_hold, ).result() except CopySourceTooBig as e: copy_source.length = e.size progress_listener = DoNothingProgressListener() logger.warning( 'a copy of large object of unknown size is upgraded to the large file interface. No progress report will be provided.' ) return self.create_file( [WriteIntent(copy_source)], new_file_name, content_type=content_type, file_info=file_info, progress_listener=progress_listener, encryption=destination_encryption, file_retention=file_retention, legal_hold=legal_hold, cache_control=cache_control, min_part_size=min_part_size, max_part_size=max_part_size, ) def delete_file_version(self, file_id: str, file_name: str, bypass_governance: bool = False): """ Delete a file version. :param file_id: a file ID :param file_name: a file name :param bypass_governance: Must be set to true if deleting a file version protected by Object Lock governance mode retention settings (unless its retention period expired) """ # filename argument is not first, because one day it may become optional return self.api.delete_file_version(file_id, file_name, bypass_governance) @disable_trace def as_dict(self): """ Return bucket representation as a dictionary. :rtype: dict """ result = { 'accountId': self.api.account_info.get_account_id(), 'bucketId': self.id_, } if self.name is not None: result['bucketName'] = self.name if self.type_ is not None: result['bucketType'] = self.type_ result['bucketInfo'] = self.bucket_info result['corsRules'] = self.cors_rules result['lifecycleRules'] = self.lifecycle_rules result['revision'] = self.revision result['options'] = self.options_set result['defaultServerSideEncryption'] = self.default_server_side_encryption.as_dict() result['isFileLockEnabled'] = self.is_file_lock_enabled result['defaultRetention'] = self.default_retention.as_dict() result['replication'] = self.replication and self.replication.as_dict() return result def __repr__(self): return f'Bucket<{self.id_},{self.name},{self.type_}>' def get_notification_rules(self) -> list[NotificationRuleResponse]: """ Get all notification rules for this bucket. """ return self.api.session.get_bucket_notification_rules(self.id_) def set_notification_rules( self, rules: Iterable[NotificationRule] ) -> list[NotificationRuleResponse]: """ Set notification rules for this bucket. """ return self.api.session.set_bucket_notification_rules(self.id_, rules) class BucketFactory: """ This is a factory for creating bucket objects from different kind of objects. """ BUCKET_CLASS = staticmethod(Bucket) @classmethod def from_api_response(cls, api, response): """ Create a Bucket object from API response. :param b2sdk.v2.B2Api api: API object :param requests.Response response: response object :rtype: b2sdk.v2.Bucket """ return [cls.from_api_bucket_dict(api, bucket_dict) for bucket_dict in response['buckets']] @classmethod def from_api_bucket_dict(cls, api, bucket_dict): """ Turn a dictionary, like this: .. code-block:: python { "bucketType": "allPrivate", "bucketId": "a4ba6a39d8b6b5fd561f0010", "bucketName": "zsdfrtsazsdfafr", "accountId": "4aa9865d6f00", "bucketInfo": {}, "options": [], "revision": 1, "defaultServerSideEncryption": { "isClientAuthorizedToRead" : true, "value": { "algorithm" : "AES256", "mode" : "SSE-B2" } }, "fileLockConfiguration": { "isClientAuthorizedToRead": true, "value": { "defaultRetention": { "mode": null, "period": null }, "isFileLockEnabled": false } }, "replicationConfiguration": { "clientIsAllowedToRead": true, "value": { "asReplicationSource": { "replicationRules": [ { "destinationBucketId": "c5f35d53a90a7ea284fb0719", "fileNamePrefix": "", "includeExistingFiles": True, "isEnabled": true, "priority": 1, "replicationRuleName": "replication-us-west" }, { "destinationBucketId": "55f34d53a96a7ea284fb0719", "fileNamePrefix": "", "includeExistingFiles": True, "isEnabled": true, "priority": 2, "replicationRuleName": "replication-us-west-2" } ], "sourceApplicationKeyId": "10053d55ae26b790000000006" }, "asReplicationDestination": { "sourceToDestinationKeyMapping": { "10053d55ae26b790000000045": "10053d55ae26b790000000004", "10053d55ae26b790000000046": "10053d55ae26b790030000004" } } } } } into a Bucket object. :param b2sdk.v2.B2Api api: API client :param dict bucket_dict: a dictionary with bucket properties :rtype: b2sdk.v2.Bucket """ type_ = bucket_dict['bucketType'] if type_ is None: raise UnrecognizedBucketType(bucket_dict['bucketType']) bucket_name = bucket_dict['bucketName'] bucket_id = bucket_dict['bucketId'] bucket_info = bucket_dict['bucketInfo'] cors_rules = bucket_dict['corsRules'] lifecycle_rules = bucket_dict['lifecycleRules'] revision = bucket_dict['revision'] options = set(bucket_dict['options']) if 'defaultServerSideEncryption' not in bucket_dict: raise UnexpectedCloudBehaviour('server did not provide `defaultServerSideEncryption`') default_server_side_encryption = EncryptionSettingFactory.from_bucket_dict(bucket_dict) file_lock_configuration = FileLockConfiguration.from_bucket_dict(bucket_dict) replication = ReplicationConfigurationFactory.from_bucket_dict(bucket_dict).value return cls.BUCKET_CLASS( api, bucket_id, bucket_name, type_, bucket_info, cors_rules, lifecycle_rules, revision, bucket_dict, options, default_server_side_encryption, file_lock_configuration.default_retention, file_lock_configuration.is_file_lock_enabled, replication, ) b2-sdk-python-2.8.0/b2sdk/_internal/cache.py000066400000000000000000000072421474454370000205340ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/cache.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING if TYPE_CHECKING: from b2sdk._internal.account_info.abstract import AbstractAccountInfo class AbstractCache(metaclass=ABCMeta): def clear(self): self.set_bucket_name_cache(tuple()) @abstractmethod def get_bucket_id_or_none_from_bucket_name(self, name): pass @abstractmethod def get_bucket_name_or_none_from_allowed(self): pass @abstractmethod def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> str | None: pass @abstractmethod def list_bucket_names_ids(self) -> list[tuple[str, str]]: """ List buckets in the cache. :return: list of tuples (bucket_name, bucket_id) """ @abstractmethod def save_bucket(self, bucket): pass @abstractmethod def set_bucket_name_cache(self, buckets): pass def _name_id_iterator(self, buckets): return ((bucket.name, bucket.id_) for bucket in buckets) class DummyCache(AbstractCache): """ A cache that does nothing. """ def get_bucket_id_or_none_from_bucket_name(self, name): return None def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> str | None: return None def get_bucket_name_or_none_from_allowed(self): return None def list_bucket_names_ids(self) -> list[tuple[str, str]]: return [] def save_bucket(self, bucket): pass def set_bucket_name_cache(self, buckets): pass class InMemoryCache(AbstractCache): """ A cache that stores the information in memory. """ def __init__(self): self.name_id_map = {} self.bucket_name = None def get_bucket_id_or_none_from_bucket_name(self, name): return self.name_id_map.get(name) def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> str | None: for name, cached_id_ in self.name_id_map.items(): if cached_id_ == bucket_id: return name return None def get_bucket_name_or_none_from_allowed(self): return self.bucket_name def list_bucket_names_ids(self) -> list[tuple[str, str]]: return sorted(tuple(item) for item in self.name_id_map.items()) def save_bucket(self, bucket): self.name_id_map[bucket.name] = bucket.id_ def set_bucket_name_cache(self, buckets): self.name_id_map = dict(self._name_id_iterator(buckets)) class AuthInfoCache(AbstractCache): """ A cache that stores data persistently in StoredAccountInfo. """ def __init__(self, info: AbstractAccountInfo): self.info = info def get_bucket_id_or_none_from_bucket_name(self, name): return self.info.get_bucket_id_or_none_from_bucket_name(name) def get_bucket_name_or_none_from_bucket_id(self, bucket_id) -> str | None: return self.info.get_bucket_name_or_none_from_bucket_id(bucket_id) def get_bucket_name_or_none_from_allowed(self): return self.info.get_bucket_name_or_none_from_allowed() def list_bucket_names_ids(self) -> list[tuple[str, str]]: return self.info.list_bucket_names_ids() def save_bucket(self, bucket): self.info.save_bucket(bucket) def set_bucket_name_cache(self, buckets): self.info.refresh_entire_bucket_name_cache(self._name_id_iterator(buckets)) b2-sdk-python-2.8.0/b2sdk/_internal/encryption/000077500000000000000000000000001474454370000213045ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/encryption/__init__.py000066400000000000000000000005241474454370000234160ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/encryption/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/encryption/setting.py000066400000000000000000000305461474454370000233430ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/encryption/setting.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import enum import logging import urllib import urllib.parse from ..http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME, SSE_C_KEY_ID_HEADER from ..utils import b64_of_bytes, md5_of_bytes from .types import ( ENCRYPTION_MODES_WITH_MANDATORY_ALGORITHM, ENCRYPTION_MODES_WITH_MANDATORY_KEY, EncryptionAlgorithm, EncryptionMode, ) logger = logging.getLogger(__name__) class _UnknownKeyId(enum.Enum): """The purpose of this enum is to provide a sentinel that can be used with type annotations.""" unknown_key_id = 0 UNKNOWN_KEY_ID = _UnknownKeyId.unknown_key_id """ Value for EncryptionKey.key_id that signifies that the key id may or may not be defined. Useful when establishing encryption settings based on user input (and not based on B2 cloud data). """ class EncryptionKey: """ Hold information about encryption key: the key itself, and its id. The id may be None, if it's not set in encrypted file's fileInfo, or UNKNOWN_KEY_ID when that information is missing. The secret may be None, if encryption metadata is read from the server. """ SECRET_REPR = '******' def __init__(self, secret: bytes | None, key_id: str | None | _UnknownKeyId): self.secret = secret self.key_id = key_id def __eq__(self, other): return self.secret == other.secret and self.key_id == other.key_id def __repr__(self): key_repr = self.SECRET_REPR if self.secret is None: key_repr = None if self.key_id is UNKNOWN_KEY_ID: key_id_repr = 'unknown' else: key_id_repr = repr(self.key_id) return f'<{self.__class__.__name__}({key_repr}, {key_id_repr})>' def as_dict(self): """ Dump EncryptionKey as dict for serializing a to json for requests. """ if self.secret is not None: return { 'customerKey': self.key_b64(), 'customerKeyMd5': self.key_md5(), } return { 'customerKey': self.SECRET_REPR, 'customerKeyMd5': self.SECRET_REPR, } def key_b64(self): return b64_of_bytes(self.secret) def key_md5(self): return b64_of_bytes(md5_of_bytes(self.secret)) class EncryptionSetting: """ Hold information about encryption mode, algorithm and key (for bucket default, file version info or even upload) """ def __init__( self, mode: EncryptionMode, algorithm: EncryptionAlgorithm = None, key: EncryptionKey = None, ): """ :param b2sdk.v2.EncryptionMode mode: encryption mode :param b2sdk.v2.EncryptionAlgorithm algorithm: encryption algorithm :param b2sdk.v2.EncryptionKey key: encryption key object for SSE-C """ self.mode = mode self.algorithm = algorithm self.key = key if self.mode == EncryptionMode.NONE and (self.algorithm or self.key): raise ValueError("cannot specify algorithm or key for 'plaintext' encryption mode") if self.mode in ENCRYPTION_MODES_WITH_MANDATORY_ALGORITHM and not self.algorithm: raise ValueError(f'must specify algorithm for encryption mode {self.mode}') if self.mode in ENCRYPTION_MODES_WITH_MANDATORY_KEY and not self.key: raise ValueError( f'must specify key for encryption mode {self.mode} and algorithm {self.algorithm}' ) def __eq__(self, other): if other is None: raise ValueError('cannot compare a known encryption setting to an unknown one') return ( self.mode == other.mode and self.algorithm == other.algorithm and self.key == other.key ) def serialize_to_json_for_request(self): if self.key and self.key.secret is None: raise ValueError('cannot use an unknown key in requests') return self.as_dict() def as_dict(self): """ Represent the setting as a dict, for example: .. code-block:: python { 'mode': 'SSE-C', 'algorithm': 'AES256', 'customerKey': 'U3hWbVlxM3Q2djl5JEImRSlIQE1jUWZUalduWnI0dTc=', 'customerKeyMd5': 'SWx9GFv5BTT1jdwf48Bx+Q==' } .. code-block:: python { 'mode': 'SSE-B2', 'algorithm': 'AES256' } or .. code-block:: python { 'mode': 'none' } """ result = {'mode': self.mode.value} if self.algorithm is not None: result['algorithm'] = self.algorithm.value if self.mode == EncryptionMode.SSE_C: result.update(self.key.as_dict()) return result def add_to_upload_headers(self, headers): if self.mode == EncryptionMode.NONE: # as of 2021-03-16, server always fails it headers['X-Bz-Server-Side-Encryption'] = self.mode.name elif self.mode == EncryptionMode.SSE_B2: self._add_sse_b2_headers(headers) elif self.mode == EncryptionMode.SSE_C: if self.key.key_id is UNKNOWN_KEY_ID: raise ValueError('Cannot upload a file with an unknown key id') self._add_sse_c_headers(headers) if self.key.key_id is not None: header = SSE_C_KEY_ID_HEADER if headers.get(header) is not None and headers[header] != self.key.key_id: raise ValueError( f'Ambiguous key id set: "{headers[header]}" in headers and "{self.key.key_id}" in {self.__class__.__name__}' ) headers[header] = urllib.parse.quote(str(self.key.key_id)) else: raise NotImplementedError(f'unsupported encryption setting: {self}') def add_to_download_headers(self, headers): if self.mode == EncryptionMode.NONE: return elif self.mode == EncryptionMode.SSE_B2: self._add_sse_b2_headers(headers) elif self.mode == EncryptionMode.SSE_C: self._add_sse_c_headers(headers) else: raise NotImplementedError(f'unsupported encryption setting: {self}') def _add_sse_b2_headers(self, headers): headers['X-Bz-Server-Side-Encryption'] = self.algorithm.name def _add_sse_c_headers(self, headers): if self.key.secret is None: raise ValueError('Cannot use an unknown key in http headers') headers['X-Bz-Server-Side-Encryption-Customer-Algorithm'] = self.algorithm.name headers['X-Bz-Server-Side-Encryption-Customer-Key'] = self.key.key_b64() headers['X-Bz-Server-Side-Encryption-Customer-Key-Md5'] = self.key.key_md5() def add_key_id_to_file_info(self, file_info: dict | None): if self.key is None or self.key.key_id is None: return file_info if self.key.key_id is UNKNOWN_KEY_ID: raise ValueError('Cannot add an unknown key id to file info') if file_info is None: file_info = {} if ( file_info.get(SSE_C_KEY_ID_FILE_INFO_KEY_NAME) is not None and file_info[SSE_C_KEY_ID_FILE_INFO_KEY_NAME] != self.key.key_id ): raise ValueError( f'Ambiguous key id set: "{file_info[SSE_C_KEY_ID_FILE_INFO_KEY_NAME]}" in file_info and "{self.key.key_id}" in {self.__class__.__name__}' ) file_info[SSE_C_KEY_ID_FILE_INFO_KEY_NAME] = self.key.key_id return file_info def __repr__(self): return f'<{self.__class__.__name__}({self.mode}, {self.algorithm}, {self.key})>' def is_unknown(self): return self.mode == EncryptionMode.NONE class EncryptionSettingFactory: # 2021-03-17: for the bucket the response of the server is: # if authorized to read: # "mode": "none" # or # "mode": "SSE-B2" # if not authorized to read: # isClientAuthorizedToRead is False and there is no value, so no mode # # BUT file_version (get_file_info, list_file_versions, upload_file etc) # if the file is encrypted, then # "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"}, # or # "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-C"}, # if the file is not encrypted, then "serverSideEncryption" is not present at all @classmethod def from_file_version_dict(cls, file_version_dict: dict) -> EncryptionSetting: """ Returns EncryptionSetting for the given file_version_dict retrieved from the api .. code-block:: python ... "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"}, "fileInfo": {"sse_c_key_id": "key-identifier"} ... """ sse = file_version_dict.get('serverSideEncryption') if sse is None: return EncryptionSetting(EncryptionMode.NONE) key_id = None file_info = file_version_dict.get('fileInfo') if file_info is not None and SSE_C_KEY_ID_FILE_INFO_KEY_NAME in file_info: key_id = urllib.parse.unquote(file_info[SSE_C_KEY_ID_FILE_INFO_KEY_NAME]) return cls._from_value_dict(sse, key_id=key_id) @classmethod def from_bucket_dict(cls, bucket_dict: dict) -> EncryptionSetting | None: """ Returns EncryptionSetting for the given bucket dict retrieved from the api, or None if unauthorized Example inputs: .. code-block:: python ... "defaultServerSideEncryption": { "isClientAuthorizedToRead" : true, "value": { "algorithm" : "AES256", "mode" : "SSE-B2" } } ... unset: .. code-block:: python ... "defaultServerSideEncryption": { "isClientAuthorizedToRead" : true, "value": { "mode" : "none" } } ... unknown: .. code-block:: python ... "defaultServerSideEncryption": { "isClientAuthorizedToRead" : false } ... """ default_sse = bucket_dict.get( 'defaultServerSideEncryption', {'isClientAuthorizedToRead': False}, ) if not default_sse['isClientAuthorizedToRead']: return EncryptionSetting(EncryptionMode.UNKNOWN) assert 'value' in default_sse return cls._from_value_dict(default_sse['value']) @classmethod def _from_value_dict(cls, value_dict, key_id=None): kwargs = {} if value_dict is None: kwargs['mode'] = EncryptionMode.NONE else: mode = EncryptionMode(value_dict['mode'] or 'none') kwargs['mode'] = mode algorithm = value_dict.get('algorithm') if algorithm is not None: kwargs['algorithm'] = EncryptionAlgorithm(algorithm) if mode == EncryptionMode.SSE_C: kwargs['key'] = EncryptionKey(key_id=key_id, secret=None) return EncryptionSetting(**kwargs) @classmethod def from_response_headers(cls, headers): if 'X-Bz-Server-Side-Encryption' in headers: mode = EncryptionMode.SSE_B2 algorithm = EncryptionAlgorithm(headers['X-Bz-Server-Side-Encryption']) return EncryptionSetting(mode, algorithm) if 'X-Bz-Server-Side-Encryption-Customer-Algorithm' in headers: mode = EncryptionMode.SSE_C algorithm = EncryptionAlgorithm( headers['X-Bz-Server-Side-Encryption-Customer-Algorithm'] ) key_id = headers.get(SSE_C_KEY_ID_HEADER) key = EncryptionKey(secret=None, key_id=key_id) return EncryptionSetting(mode, algorithm, key) return EncryptionSetting(EncryptionMode.NONE) SSE_NONE = EncryptionSetting( mode=EncryptionMode.NONE, ) """ Commonly used "no encryption" setting """ SSE_B2_AES = EncryptionSetting( mode=EncryptionMode.SSE_B2, algorithm=EncryptionAlgorithm.AES256, ) """ Commonly used SSE-B2 setting """ b2-sdk-python-2.8.0/b2sdk/_internal/encryption/types.py000066400000000000000000000025721474454370000230300ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/encryption/types.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from enum import Enum, unique @unique class EncryptionAlgorithm(Enum): """Encryption algorithm.""" AES256 = 'AES256' def get_length(self) -> int: if self is EncryptionAlgorithm.AES256: return int(256 / 8) raise NotImplementedError() @unique class EncryptionMode(Enum): """Encryption mode.""" UNKNOWN = None #: unknown encryption mode (sdk doesn't know or used key has no rights to know) NONE = 'none' #: no encryption (plaintext) SSE_B2 = 'SSE-B2' #: server-side encryption with key maintained by B2 SSE_C = 'SSE-C' #: server-side encryption with key provided by the client # CLIENT = 'CLIENT' #: client-side encryption def can_be_set_as_bucket_default(self): return self in BUCKET_DEFAULT_ENCRYPTION_MODES ENCRYPTION_MODES_WITH_MANDATORY_ALGORITHM = frozenset((EncryptionMode.SSE_B2, EncryptionMode.SSE_C)) ENCRYPTION_MODES_WITH_MANDATORY_KEY = frozenset((EncryptionMode.SSE_C,)) BUCKET_DEFAULT_ENCRYPTION_MODES = frozenset((EncryptionMode.NONE, EncryptionMode.SSE_B2)) b2-sdk-python-2.8.0/b2sdk/_internal/exception.py000066400000000000000000000517031474454370000214700ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging import re import typing import warnings from abc import ABCMeta from typing import Any from .utils import camelcase_to_underscore, trace_call UPLOAD_TOKEN_USED_CONCURRENTLY_ERROR_MESSAGE_RE = re.compile( r'^more than one upload using auth token (?P[^)]+)$' ) COPY_SOURCE_TOO_BIG_ERROR_MESSAGE_RE = re.compile(r'^Copy source too big: (?P[\d]+)$') logger = logging.getLogger(__name__) class B2Error(Exception, metaclass=ABCMeta): def __init__(self, *args, **kwargs): """ Python 2 does not like it when you pass unicode as the message in an exception. We like to use file names in exception messages. To avoid problems, if the message has any non-ascii characters in it, they are replaced with backslash-uNNNN. https://pythonhosted.org/kitchen/unicode-frustrations.html#frustration-5-exceptions """ # If the exception is caused by a b2 server response, # the server MAY have included instructions to pause the thread before issuing any more requests self.retry_after_seconds = None super().__init__(*args, **kwargs) @property def prefix(self): """ Nice, auto-generated error message prefix. >>> B2SimpleError().prefix 'Simple error' >>> AlreadyFailed().prefix 'Already failed' """ prefix = self.__class__.__name__ if prefix.startswith('B2'): prefix = prefix[2:] prefix = camelcase_to_underscore(prefix).replace('_', ' ') return prefix[0].upper() + prefix[1:] def should_retry_http(self): """ Return true if this is an error that can cause an HTTP call to be retried. """ return False def should_retry_upload(self): """ Return true if this is an error that should tell the upload code to get a new upload URL and try the upload again. """ return False class InvalidUserInput(B2Error): pass class B2SimpleError(B2Error, metaclass=ABCMeta): """ A B2Error with a message prefix. """ def __str__(self): return f'{self.prefix}: {super().__str__()}' class NotAllowedByAppKeyError(B2SimpleError, metaclass=ABCMeta): """ Base class for errors caused by restrictions on an application key. """ class TransientErrorMixin(metaclass=ABCMeta): def should_retry_http(self): return True def should_retry_upload(self): return True class AlreadyFailed(B2SimpleError): pass class BadDateFormat(B2SimpleError): prefix = 'Date from server' class BadFileInfo(B2SimpleError): pass class BadJson(B2SimpleError): prefix = 'Bad request' class BadUploadUrl(B2SimpleError): def should_retry_upload(self): return True class BrokenPipe(B2Error): def __str__(self): return 'Broken pipe: unable to send entire request' def should_retry_upload(self): return True class CapabilityNotAllowed(NotAllowedByAppKeyError): pass class ChecksumMismatch(TransientErrorMixin, B2Error): def __init__(self, checksum_type, expected, actual): super().__init__() self.checksum_type = checksum_type self.expected = expected self.actual = actual def __str__(self): return f'{self.checksum_type} checksum mismatch -- bad data' class B2HttpCallbackException(B2SimpleError): pass class B2HttpCallbackPostRequestException(B2HttpCallbackException): pass class B2HttpCallbackPreRequestException(B2HttpCallbackException): pass class BucketNotAllowed(NotAllowedByAppKeyError): pass class ClockSkew(B2HttpCallbackPostRequestException): """ The clock on the server differs from the local clock by too much. """ def __init__(self, clock_skew_seconds): """ :param int clock_skew_seconds: The difference: local_clock - server_clock """ super().__init__() self.clock_skew_seconds = clock_skew_seconds def __str__(self): if self.clock_skew_seconds < 0: return 'ClockSkew: local clock is %d seconds behind server' % ( -self.clock_skew_seconds, ) else: return 'ClockSkew; local clock is %d seconds ahead of server' % ( self.clock_skew_seconds, ) class Conflict(B2SimpleError): pass class ConnectionReset(B2Error): def __str__(self): return 'Connection reset' def should_retry_upload(self): return True class B2ConnectionError(TransientErrorMixin, B2SimpleError): pass class B2RequestTimeout(TransientErrorMixin, B2SimpleError): pass class B2RequestTimeoutDuringUpload(B2RequestTimeout): # if a timeout is hit during upload, it is not guaranteed that the the server has released the upload token lock already, so we'll use a new token def should_retry_http(self): return False class DestFileNewer(B2Error): def __init__(self, dest_path, source_path, dest_prefix, source_prefix): super().__init__() self.dest_path = dest_path self.source_path = source_path self.dest_prefix = dest_prefix self.source_prefix = source_prefix def __str__(self): return f'source file is older than destination: {self.source_prefix}{self.source_path.relative_path} with a time of {self.source_path.mod_time} cannot be synced to {self.dest_prefix}{self.dest_path.relative_path} with a time of {self.dest_path.mod_time}, unless a valid newer_file_mode is provided' def should_retry_http(self): return True class DuplicateBucketName(B2SimpleError): prefix = 'Bucket name is already in use' class ResourceNotFound(B2SimpleError): prefix = 'No such file, bucket, or endpoint' class FileOrBucketNotFound(ResourceNotFound): def __init__(self, bucket_name=None, file_id_or_name=None): super().__init__() self.bucket_name = bucket_name self.file_id_or_name = file_id_or_name def __str__(self): file_str = ('file [%s]' % self.file_id_or_name) if self.file_id_or_name else 'a file' bucket_str = ('bucket [%s]' % self.bucket_name) if self.bucket_name else 'a bucket' return f'Could not find {file_str} within {bucket_str}' class BucketIdNotFound(ResourceNotFound): def __init__(self, bucket_id): self.bucket_id = bucket_id def __str__(self): return f'Bucket with id={self.bucket_id} not found' class FileAlreadyHidden(B2SimpleError): pass class FileNotHidden(B2SimpleError): prefix = 'File not hidden' class FileDeleted(B2SimpleError): prefix = 'File deleted' class UnexpectedFileVersionAction(B2SimpleError): prefix = 'Unexpected file version action returned by the server' class FileNameNotAllowed(NotAllowedByAppKeyError): pass class FileNotPresent(FileOrBucketNotFound): def __str__(self): # overridden to retain message across prev versions return 'File not present%s' % (': ' + self.file_id_or_name if self.file_id_or_name else '') class UnusableFileName(B2SimpleError): """ Raise when a filename doesn't meet the rules. Could possibly use InvalidUploadSource, but this is intended for the filename on the server, which could differ. https://www.backblaze.com/b2/docs/files.html. """ pass class InvalidMetadataDirective(B2Error): pass class SSECKeyIdMismatchInCopy(InvalidMetadataDirective): pass class InvalidRange(B2Error): def __init__(self, content_length, range_): super().__init__() self.content_length = content_length self.range_ = range_ def __str__(self): return ( 'A range of %d-%d was requested (size of %d), but cloud could only serve %d of that' % ( self.range_[0], self.range_[1], self.range_[1] - self.range_[0] + 1, self.content_length, ) ) class InvalidUploadSource(B2SimpleError): pass class BadRequest(B2Error): def __init__(self, message, code): super().__init__() self.message = message self.code = code def __str__(self): return f'{self.message} ({self.code})' class CopySourceTooBig(BadRequest): def __init__(self, message, code, size: int): super().__init__(message, code) self.size = size class Unauthorized(B2Error): def __init__(self, message, code): super().__init__() self.message = message self.code = code def __str__(self): return f'{self.message} ({self.code})' def should_retry_upload(self): return True class EmailNotVerified(Unauthorized): def should_retry_upload(self): return False class NoPaymentHistory(Unauthorized): def should_retry_upload(self): return False class InvalidAuthToken(Unauthorized): """ Specific type of Unauthorized that means the auth token is invalid. This is not the case where the auth token is valid, but does not allow access. """ def __init__(self, message, code): super().__init__('Invalid authorization token. Server said: ' + message, code) class RestrictedBucket(B2Error): def __init__(self, bucket_name): super().__init__() self.bucket_name = bucket_name def __str__(self): return 'Application key is restricted to bucket: %s' % self.bucket_name class RestrictedBucketMissing(RestrictedBucket): def __init__(self): super().__init__('') def __str__(self): return "Application key is restricted to a bucket that doesn't exist" class MaxFileSizeExceeded(B2Error): def __init__(self, size, max_allowed_size): super().__init__() self.size = size self.max_allowed_size = max_allowed_size def __str__(self): return f'Allowed file size of exceeded: {self.size} > {self.max_allowed_size}' class MaxRetriesExceeded(B2Error): def __init__(self, limit, exception_info_list): super().__init__() self.limit = limit self.exception_info_list = exception_info_list def __str__(self): exceptions = '\n'.join(str(wrapped_error) for wrapped_error in self.exception_info_list) return f'FAILED to upload after {self.limit} tries. Encountered exceptions: {exceptions}' class MissingPart(B2SimpleError): prefix = 'Part number has not been uploaded' class NonExistentBucket(FileOrBucketNotFound): def __str__(self): # overridden to retain message across prev versions return 'No such bucket%s' % (': ' + self.bucket_name if self.bucket_name else '') class FileSha1Mismatch(B2SimpleError): prefix = 'Upload file SHA1 mismatch' class PartSha1Mismatch(B2Error): def __init__(self, key): super().__init__() self.key = key def __str__(self): return f'Part number {self.key} has wrong SHA1' class ServiceError(TransientErrorMixin, B2Error): """ Used for HTTP status codes 500 through 599. """ class CapExceeded(B2Error): def __str__(self): return 'Cap exceeded.' class StorageCapExceeded(CapExceeded): def __str__(self): return 'Cannot upload or copy files, storage cap exceeded.' class TransactionCapExceeded(CapExceeded): def __str__(self): return 'Cannot perform the operation, transaction cap exceeded.' class TooManyRequests(B2Error): def __init__(self, retry_after_seconds=None): super().__init__() self.retry_after_seconds = retry_after_seconds def __str__(self): return 'Too many requests' def should_retry_http(self): return True class TruncatedOutput(TransientErrorMixin, B2Error): def __init__(self, bytes_read, file_size): super().__init__() self.bytes_read = bytes_read self.file_size = file_size def __str__(self): return 'only %d of %d bytes read' % ( self.bytes_read, self.file_size, ) class UnexpectedCloudBehaviour(B2SimpleError): pass class UnknownError(B2SimpleError): pass class UnknownHost(B2Error): def __str__(self): return 'unknown host' class UnrecognizedBucketType(B2Error): pass class UnsatisfiableRange(B2Error): def __str__(self): return 'The range in the request is outside the size of the file' class UploadTokenUsedConcurrently(B2Error): def __init__(self, token): super().__init__() self.token = token def __str__(self): return f'More than one concurrent upload using auth token {self.token}' class AccessDenied(B2Error): def __str__(self): return 'This call with these parameters is not allowed for this auth token' class SSECKeyError(AccessDenied): def __str__(self): return 'Wrong or no SSE-C key provided when reading a file.' class RetentionWriteError(AccessDenied): def __str__(self): return ( "Auth token not authorized to write retention or file already in 'compliance' mode or " 'bypassGovernance=true parameter missing' ) class WrongEncryptionModeForBucketDefault(InvalidUserInput): def __init__(self, encryption_mode): super().__init__() self.encryption_mode = encryption_mode def __str__(self): return f'{self.encryption_mode} cannot be used as default for a bucket.' class CopyArgumentsMismatch(InvalidUserInput): pass class DisablingFileLockNotSupported(B2Error): def __str__(self): return 'Disabling file lock is not supported' class SourceReplicationConflict(B2Error): def __str__(self): return 'Operation not supported for buckets with source replication' class EnablingFileLockOnRestrictedBucket(B2Error): def __str__(self): return 'Turning on file lock for a restricted bucket is not allowed' class InvalidJsonResponse(B2SimpleError): UP_TO_BYTES_COUNT = 200 def __init__(self, content: bytes): self.content = content message = self.content[: self.UP_TO_BYTES_COUNT].decode('utf-8', errors='replace') if len(self.content) > self.UP_TO_BYTES_COUNT: message += '...' super().__init__(message) class PotentialS3EndpointPassedAsRealm(InvalidJsonResponse): pass class DestinationError(B2Error): pass class DestinationDirectoryError(DestinationError): pass class DestinationDirectoryDoesntExist(DestinationDirectoryError): pass class DestinationParentIsNotADirectory(DestinationDirectoryError): pass class DestinationIsADirectory(DestinationDirectoryError): pass class DestinationDirectoryDoesntAllowOperation(DestinationDirectoryError): pass class EventTypeError(BadRequest): pass class EventTypeCategoriesError(EventTypeError): pass class EventTypeOverlapError(EventTypeError): pass class EventTypesEmptyError(EventTypeError): pass class EventTypeInvalidError(EventTypeError): pass def _event_type_invalid_error(code: str, message: str, **_) -> B2Error: from b2sdk._internal.raw_api import EVENT_TYPE valid_types = sorted(typing.get_args(EVENT_TYPE)) return EventTypeInvalidError( f'Event Type error: {message!r}. Valid types: {sorted(valid_types)!r}', code ) _error_handlers: dict[tuple[int, str | None], typing.Callable] = { (400, 'event_type_categories'): lambda code, message, **_: EventTypeCategoriesError( message, code ), (400, 'event_type_overlap'): lambda code, message, **_: EventTypeOverlapError(message, code), (400, 'event_types_empty'): lambda code, message, **_: EventTypesEmptyError(message, code), (400, 'event_type_invalid'): _event_type_invalid_error, (401, 'email_not_verified'): lambda code, message, **_: EmailNotVerified(message, code), (401, 'no_payment_history'): lambda code, message, **_: NoPaymentHistory(message, code), } @trace_call(logger) def interpret_b2_error( status: int, code: str | None, message: str | None, response_headers: dict[str, Any], post_params: dict[str, Any] | None = None, ) -> B2Error: post_params = post_params or {} handler = _error_handlers.get((status, code)) if handler: error = handler( status=status, code=code, message=message, response_headers=response_headers, post_params=post_params, ) if error: return error if status == 400 and code == 'already_hidden': return FileAlreadyHidden(post_params.get('fileName')) elif status == 400 and code == 'bad_json': return BadJson(message) elif (status == 400 and code in ('no_such_file', 'file_not_present')) or ( status == 404 and code == 'not_found' ): # hide_file returns 400 and "no_such_file" # delete_file_version returns 400 and "file_not_present" # get_file_info returns 404 and "not_found" # download_file_by_name/download_file_by_id return 404 and "not_found" # but don't have post_params return FileNotPresent( file_id_or_name=post_params.get('fileId') or post_params.get('fileName') ) elif status == 404: # often times backblaze will return cryptic error messages on invalid URLs. # We should ideally only reach that case on programming error or outdated # sdk versions, but to prevent user confusion we omit the message param return ResourceNotFound() elif status == 400 and code == 'duplicate_bucket_name': return DuplicateBucketName(post_params.get('bucketName')) elif status == 400 and code == 'missing_part': return MissingPart(post_params.get('fileId')) elif status == 400 and code == 'part_sha1_mismatch': return PartSha1Mismatch(post_params.get('fileId')) elif status == 400 and code == 'bad_bucket_id': return BucketIdNotFound(post_params.get('bucketId')) elif status == 400 and code == 'auth_token_limit': matcher = UPLOAD_TOKEN_USED_CONCURRENTLY_ERROR_MESSAGE_RE.match(message) assert matcher is not None, f'unexpected error message: {message}' token = matcher.group('token') return UploadTokenUsedConcurrently(token) elif status == 400 and code == 'source_too_large': matcher = COPY_SOURCE_TOO_BIG_ERROR_MESSAGE_RE.match(message) assert matcher is not None, f'unexpected error message: {message}' size = int(matcher.group('size')) return CopySourceTooBig(message, code, size) elif status == 400 and code == 'file_lock_conflict': return DisablingFileLockNotSupported() elif status == 400 and code == 'source_replication_conflict': return SourceReplicationConflict() elif status == 400 and code == 'restricted_bucket_conflict': return EnablingFileLockOnRestrictedBucket() elif status == 400 and code == 'bad_request': # it's "bad_request" on 2022-09-14, but will become 'disabling_file_lock_not_allowed' # TODO: cleanup after 2022-09-22 if ( message == 'fileLockEnabled value of false is not allowed when bucket is already file lock enabled.' ): return DisablingFileLockNotSupported() # it's "bad_request" on 2022-09-14, but will become 'source_replication_conflict' # TODO: cleanup after 2022-09-22 if ( message == 'Turning on file lock for an existing bucket having source replication configuration is not allowed.' ): return SourceReplicationConflict() # it's "bad_request" on 2022-09-14, but will become 'restricted_bucket_conflict' # TODO: cleanup after 2022-09-22 if message == 'Turning on file lock for a restricted bucket is not allowed.': return EnablingFileLockOnRestrictedBucket() return BadRequest(message, code) elif status == 400: warnings.warn( f'bad request exception with an unknown `code`. message={message}, code={code}' ) return BadRequest(message, code) elif status == 401 and code in ('bad_auth_token', 'expired_auth_token'): return InvalidAuthToken(message, code) elif status == 401: return Unauthorized(message, code) elif status == 403 and code == 'storage_cap_exceeded': return StorageCapExceeded() elif status == 403 and code == 'transaction_cap_exceeded': return TransactionCapExceeded() elif status == 403 and code == 'access_denied': return AccessDenied() elif status == 409: return Conflict() elif status == 416 and code == 'range_not_satisfiable': return UnsatisfiableRange() elif status == 429: return TooManyRequests(retry_after_seconds=response_headers.get('retry-after')) elif 500 <= status < 600: return ServiceError('%d %s %s' % (status, code, message)) return UnknownError('%d %s %s' % (status, code, message)) b2-sdk-python-2.8.0/b2sdk/_internal/file_lock.py000066400000000000000000000327571474454370000214310ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/file_lock.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import enum from .exception import UnexpectedCloudBehaviour ACTIONS_WITHOUT_LOCK_SETTINGS = frozenset(['hide', 'folder']) @enum.unique class RetentionMode(enum.Enum): """Enum class representing retention modes set in files and buckets""" GOVERNANCE = 'governance' #: retention settings for files in this mode can be modified by clients with appropriate application key capabilities COMPLIANCE = 'compliance' #: retention settings for files in this mode can only be modified by extending the retention dates by clients with appropriate application key capabilities NONE = None #: retention not set UNKNOWN = 'unknown' #: the client is not authorized to read retention settings RETENTION_MODES_REQUIRING_PERIODS = frozenset({RetentionMode.COMPLIANCE, RetentionMode.GOVERNANCE}) class RetentionPeriod: """Represent a time period (either in days or in years) that is used as a default for bucket retention""" KNOWN_UNITS = ['days', 'years'] def __init__(self, years: int | None = None, days: int | None = None): """Create a retention period, provide exactly one of: days, years""" assert (years is None) != (days is None) if years is not None: self.duration = years self.unit = 'years' else: self.duration = days self.unit = 'days' @classmethod def from_period_dict(cls, period_dict): """ Build a RetentionPeriod from an object returned by the server, such as: .. code-block :: { "duration": 2, "unit": "years" } """ assert period_dict['unit'] in cls.KNOWN_UNITS return cls(**{period_dict['unit']: period_dict['duration']}) def as_dict(self): return { 'duration': self.duration, 'unit': self.unit, } def __repr__(self): return f'{self.__class__.__name__}({self.duration} {self.unit})' def __eq__(self, other): return self.unit == other.unit and self.duration == other.duration class FileRetentionSetting: """Represent file retention settings, i.e. whether the file is retained, in which mode and until when""" def __init__(self, mode: RetentionMode, retain_until: int | None = None): if mode in RETENTION_MODES_REQUIRING_PERIODS and retain_until is None: raise ValueError(f'must specify retain_until for retention mode {mode}') self.mode = mode self.retain_until = retain_until @classmethod def from_file_version_dict(cls, file_version_dict: dict) -> FileRetentionSetting: """ Returns FileRetentionSetting for the given file_version_dict retrieved from the api. E.g. .. code-block :: { "action": "upload", "fileRetention": { "isClientAuthorizedToRead": false, "value": null }, ... } { "action": "upload", "fileRetention": { "isClientAuthorizedToRead": true, "value": { "mode": "governance", "retainUntilTimestamp": 1628942493000 } }, ... } """ if 'fileRetention' not in file_version_dict: if file_version_dict['action'] not in ACTIONS_WITHOUT_LOCK_SETTINGS: raise UnexpectedCloudBehaviour( 'No fileRetention provided for file version with action=%s' % (file_version_dict['action']) ) return NO_RETENTION_FILE_SETTING file_retention_dict = file_version_dict['fileRetention'] if not file_retention_dict['isClientAuthorizedToRead']: return cls(RetentionMode.UNKNOWN, None) return cls.from_file_retention_value_dict(file_retention_dict['value']) @classmethod def from_file_retention_value_dict( cls, file_retention_value_dict: dict ) -> FileRetentionSetting: mode = file_retention_value_dict['mode'] if mode is None: return NO_RETENTION_FILE_SETTING return cls( RetentionMode(mode), file_retention_value_dict['retainUntilTimestamp'], ) @classmethod def from_server_response(cls, server_response: dict) -> FileRetentionSetting: return cls.from_file_retention_value_dict(server_response['fileRetention']) @classmethod def from_response_headers(cls, headers) -> FileRetentionSetting: retention_mode_header = 'X-Bz-File-Retention-Mode' retain_until_header = 'X-Bz-File-Retention-Retain-Until-Timestamp' if retention_mode_header in headers: if retain_until_header in headers: retain_until = int(headers[retain_until_header]) else: retain_until = None return cls(RetentionMode(headers[retention_mode_header]), retain_until) if 'X-Bz-Client-Unauthorized-To-Read' in headers and retention_mode_header in headers[ 'X-Bz-Client-Unauthorized-To-Read' ].split(','): return UNKNOWN_FILE_RETENTION_SETTING return NO_RETENTION_FILE_SETTING # the bucket is not file-lock-enabled or the file is has no retention set def serialize_to_json_for_request(self): if self.mode is RetentionMode.UNKNOWN: raise ValueError('cannot use an unknown file retention setting in requests') return self.as_dict() def as_dict(self): return { 'mode': self.mode.value, 'retainUntilTimestamp': self.retain_until, } def add_to_to_upload_headers(self, headers): if self.mode is RetentionMode.UNKNOWN: raise ValueError('cannot use an unknown file retention setting in requests') headers['X-Bz-File-Retention-Mode'] = str( self.mode.value ) # mode = NONE is not supported by the server at the # moment, but it should be headers['X-Bz-File-Retention-Retain-Until-Timestamp'] = str(self.retain_until) def __eq__(self, other): return self.mode == other.mode and self.retain_until == other.retain_until def __repr__(self): return f'{self.__class__.__name__}({repr(self.mode.value)}, {repr(self.retain_until)})' @enum.unique class LegalHold(enum.Enum): """Enum holding information about legalHold switch in a file.""" ON = 'on' #: legal hold set to "on" OFF = 'off' #: legal hold set to "off" UNSET = None #: server default, as for now it is functionally equivalent to OFF UNKNOWN = 'unknown' #: the client is not authorized to read legal hold settings def is_on(self): """Is the legalHold switch on?""" return self is LegalHold.ON def is_off(self): """Is the legalHold switch off or left as default (which also means off)?""" return self is LegalHold.OFF or self is LegalHold.UNSET def is_unknown(self): """Is the legalHold switch unknown?""" return self is LegalHold.UNKNOWN @classmethod def from_file_version_dict(cls, file_version_dict: dict) -> LegalHold: if 'legalHold' not in file_version_dict: if file_version_dict['action'] not in ACTIONS_WITHOUT_LOCK_SETTINGS: raise UnexpectedCloudBehaviour( 'legalHold not provided for file version with action=%s' % (file_version_dict['action']) ) return cls.UNSET if not file_version_dict['legalHold']['isClientAuthorizedToRead']: return cls.UNKNOWN return cls.from_string_or_none(file_version_dict['legalHold']['value']) @classmethod def from_server_response(cls, server_response: dict) -> LegalHold: return cls.from_string_or_none(server_response['legalHold']) @classmethod def from_string_or_none(cls, string: str | None) -> LegalHold: return cls(string) @classmethod def from_response_headers(cls, headers) -> LegalHold: legal_hold_header = 'X-Bz-File-Legal-Hold' if legal_hold_header in headers: return cls(headers['X-Bz-File-Legal-Hold']) if 'X-Bz-Client-Unauthorized-To-Read' in headers and legal_hold_header in headers[ 'X-Bz-Client-Unauthorized-To-Read' ].split(','): return cls.UNKNOWN return ( cls.UNSET ) # the bucket is not file-lock-enabled or the header is missing for any other reason def to_server(self) -> str: if self.is_unknown(): raise ValueError('Cannot use an unknown legal hold in requests') if self.is_on(): return self.__class__.ON.value return self.__class__.OFF.value def add_to_upload_headers(self, headers): headers['X-Bz-File-Legal-Hold'] = self.to_server() class BucketRetentionSetting: """Represent bucket's default file retention settings, i.e. whether the files should be retained, in which mode and for how long""" def __init__(self, mode: RetentionMode, period: RetentionPeriod | None = None): if mode in RETENTION_MODES_REQUIRING_PERIODS and period is None: raise ValueError(f'must specify period for retention mode {mode}') self.mode = mode self.period = period @classmethod def from_bucket_retention_dict(cls, retention_dict: dict): """ Build a BucketRetentionSetting from an object returned by the server, such as: .. code-block:: { "mode": "compliance", "period": { "duration": 7, "unit": "days" } } """ period = retention_dict['period'] if period is not None: period = RetentionPeriod.from_period_dict(period) return cls(RetentionMode(retention_dict['mode']), period) def as_dict(self): result = { 'mode': self.mode.value, } if self.period is not None: result['period'] = self.period.as_dict() return result def serialize_to_json_for_request(self): if self.mode == RetentionMode.UNKNOWN: raise ValueError('cannot use an unknown file lock configuration in requests') return self.as_dict() def __eq__(self, other): return self.mode == other.mode and self.period == other.period def __repr__(self): return f'{self.__class__.__name__}({repr(self.mode.value)}, {repr(self.period)})' class FileLockConfiguration: """Represent bucket's file lock configuration, i.e. whether the file lock mechanism is enabled and default file retention""" def __init__( self, default_retention: BucketRetentionSetting, is_file_lock_enabled: bool | None, ): self.default_retention = default_retention self.is_file_lock_enabled = is_file_lock_enabled @classmethod def from_bucket_dict(cls, bucket_dict): """ Build a FileLockConfiguration from an object returned by server, such as: .. code-block:: { "isClientAuthorizedToRead": true, "value": { "defaultRetention": { "mode": "governance", "period": { "duration": 2, "unit": "years" } }, "isFileLockEnabled": true } } or { "isClientAuthorizedToRead": false, "value": null } """ if not bucket_dict['fileLockConfiguration']['isClientAuthorizedToRead']: return cls(UNKNOWN_BUCKET_RETENTION, None) retention = BucketRetentionSetting.from_bucket_retention_dict( bucket_dict['fileLockConfiguration']['value']['defaultRetention'] ) is_file_lock_enabled = bucket_dict['fileLockConfiguration']['value']['isFileLockEnabled'] return cls(retention, is_file_lock_enabled) def as_dict(self): return { 'defaultRetention': self.default_retention.as_dict(), 'isFileLockEnabled': self.is_file_lock_enabled, } def __eq__(self, other): return ( self.default_retention == other.default_retention and self.is_file_lock_enabled == other.is_file_lock_enabled ) def __repr__(self): return f'{self.__class__.__name__}({repr(self.default_retention)}, {repr(self.is_file_lock_enabled)})' UNKNOWN_BUCKET_RETENTION = BucketRetentionSetting(RetentionMode.UNKNOWN) """Commonly used "unknown" default bucket retention setting""" UNKNOWN_FILE_LOCK_CONFIGURATION = FileLockConfiguration(UNKNOWN_BUCKET_RETENTION, None) """Commonly used "unknown" bucket file lock setting""" NO_RETENTION_BUCKET_SETTING = BucketRetentionSetting(RetentionMode.NONE) """Commonly used "no retention" default bucket retention""" NO_RETENTION_FILE_SETTING = FileRetentionSetting(RetentionMode.NONE) """Commonly used "no retention" file setting""" UNKNOWN_FILE_RETENTION_SETTING = FileRetentionSetting(RetentionMode.UNKNOWN) """Commonly used "unknown" file retention setting""" b2-sdk-python-2.8.0/b2sdk/_internal/file_version.py000066400000000000000000000570151474454370000221600ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/file_version.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import datetime as dt import re from copy import deepcopy from typing import TYPE_CHECKING, Any from .encryption.setting import EncryptionSetting, EncryptionSettingFactory from .file_lock import NO_RETENTION_FILE_SETTING, FileRetentionSetting, LegalHold from .http_constants import FILE_INFO_HEADER_PREFIX_LOWER, LARGE_FILE_SHA1, SRC_LAST_MODIFIED_MILLIS from .progress import AbstractProgressListener from .replication.types import ReplicationStatus from .utils import Sha1HexDigest, b2_url_decode from .utils.http_date import parse_http_date from .utils.range_ import EMPTY_RANGE, Range if TYPE_CHECKING: from .api import B2Api from .transfer.inbound.downloaded_file import DownloadedFile UNVERIFIED_CHECKSUM_PREFIX = 'unverified:' class BaseFileVersion: """ Base class for representing file metadata in B2 cloud. :ivar size - size of the whole file (for "upload" markers) """ __slots__ = [ 'id_', 'api', 'file_name', 'size', 'content_type', 'content_sha1', 'content_sha1_verified', 'file_info', 'upload_timestamp', 'server_side_encryption', 'legal_hold', 'file_retention', 'mod_time_millis', 'replication_status', ] _TYPE_MATCHER = re.compile('[a-z0-9]+_[a-z0-9]+_f([0-9]).*') _FILE_TYPE = { 1: 'small', 2: 'large', 3: 'part', 4: 'tiny', } def __init__( self, api: B2Api, id_: str, file_name: str, size: int, content_type: str | None, content_sha1: str | None, file_info: dict[str, str] | None, upload_timestamp: int, server_side_encryption: EncryptionSetting, file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, replication_status: ReplicationStatus | None = None, ): self.api = api self.id_ = id_ self.file_name = file_name self.size = size self.content_type = content_type self.content_sha1, self.content_sha1_verified = self._decode_content_sha1(content_sha1) self.file_info = file_info or {} self.upload_timestamp = upload_timestamp self.server_side_encryption = server_side_encryption self.file_retention = file_retention self.legal_hold = legal_hold self.replication_status = replication_status if SRC_LAST_MODIFIED_MILLIS in self.file_info: self.mod_time_millis = int(self.file_info[SRC_LAST_MODIFIED_MILLIS]) else: self.mod_time_millis = self.upload_timestamp @classmethod def _decode_content_sha1(cls, content_sha1): if content_sha1.startswith(UNVERIFIED_CHECKSUM_PREFIX): return content_sha1[len(UNVERIFIED_CHECKSUM_PREFIX) :], False return content_sha1, True @classmethod def _encode_content_sha1(cls, content_sha1, content_sha1_verified): if not content_sha1_verified: return f'{UNVERIFIED_CHECKSUM_PREFIX}{content_sha1}' return content_sha1 def _clone(self, **new_attributes: Any): """ Create new instance based on the old one, overriding attributes with :code:`new_attributes` (only applies to arguments passed to __init__) """ args = self._get_args_for_clone() return self.__class__(**{**args, **new_attributes}) def _get_args_for_clone(self): return { 'api': self.api, 'id_': self.id_, 'file_name': self.file_name, 'size': self.size, 'content_type': self.content_type, 'content_sha1': self._encode_content_sha1( self.content_sha1, self.content_sha1_verified ), 'file_info': self.file_info, 'upload_timestamp': self.upload_timestamp, 'server_side_encryption': self.server_side_encryption, 'file_retention': self.file_retention, 'legal_hold': self.legal_hold, 'replication_status': self.replication_status, } def as_dict(self): """represents the object as a dict which looks almost exactly like the raw api output for upload/list""" result = { 'fileId': self.id_, 'fileName': self.file_name, 'fileInfo': self.file_info, 'serverSideEncryption': self.server_side_encryption.as_dict(), 'legalHold': self.legal_hold.value, 'fileRetention': self.file_retention.as_dict(), } if self.size is not None: result['size'] = self.size if self.upload_timestamp is not None: result['uploadTimestamp'] = self.upload_timestamp if self.content_type is not None: result['contentType'] = self.content_type if self.content_sha1 is not None: result['contentSha1'] = self._encode_content_sha1( self.content_sha1, self.content_sha1_verified ) result['replicationStatus'] = self.replication_status and self.replication_status.value return result def __eq__(self, other): sentry = object() for attr in self._all_slots(): if getattr(self, attr) != getattr(other, attr, sentry): return False return True def __repr__(self): return '{}({})'.format( self.__class__.__name__, ', '.join(repr(getattr(self, attr)) for attr in self._all_slots()), ) def _all_slots(self): """Return all slots for an object (for it's class and all parent classes). Useful in auxiliary methods.""" all_slots = [] for klass in self.__class__.__mro__[-1::-1]: all_slots.extend(getattr(klass, '__slots__', [])) return all_slots def delete(self, bypass_governance: bool = False) -> FileIdAndName: """Delete this file version. bypass_governance must be set to true if deleting a file version protected by Object Lock governance mode retention settings (unless its retention period expired)""" return self.api.delete_file_version(self.id_, self.file_name, bypass_governance) def update_legal_hold(self, legal_hold: LegalHold) -> BaseFileVersion: legal_hold = self.api.update_file_legal_hold(self.id_, self.file_name, legal_hold) return self._clone(legal_hold=legal_hold) def update_retention( self, file_retention: FileRetentionSetting, bypass_governance: bool = False, ) -> BaseFileVersion: file_retention = self.api.update_file_retention( self.id_, self.file_name, file_retention, bypass_governance ) return self._clone(file_retention=file_retention) def _type(self): """ FOR TEST PURPOSES ONLY not guaranteed to work for perpetuity (using undocumented server behavior) """ m = self._TYPE_MATCHER.match(self.id_) assert m, self.id_ return self._FILE_TYPE[int(m.group(1))] def get_content_sha1(self) -> Sha1HexDigest | None: """ Get the file's content SHA1 hex digest from the header or, if its absent, from the file info. If both are missing, return None. """ if self.content_sha1 and self.content_sha1 != 'none': return self.content_sha1 elif LARGE_FILE_SHA1 in self.file_info: return Sha1HexDigest(self.file_info[LARGE_FILE_SHA1]) # content SHA1 unknown return None class FileVersion(BaseFileVersion): """ A structure which represents a version of a file (in B2 cloud). :ivar str ~.id_: ``fileId`` :ivar str ~.file_name: full file name (with path) :ivar ~.size: size in bytes, can be ``None`` (unknown) :ivar str ~.content_type: RFC 822 content type, for example ``"application/octet-stream"`` :ivar ~.upload_timestamp: in milliseconds since :abbr:`epoch (1970-01-01 00:00:00)`. Can be ``None`` (unknown). :ivar str ~.action: ``"upload"``, ``"hide"`` or ``"delete"`` """ __slots__ = [ 'account_id', 'bucket_id', 'content_md5', 'action', ] # defined at https://www.backblaze.com/b2/docs/files.html#httpHeaderSizeLimit DEFAULT_HEADERS_LIMIT = 7000 ADVANCED_HEADERS_LIMIT = 2048 def __init__( self, api: B2Api, id_: str, file_name: str, size: int | None | str, content_type: str | None, content_sha1: str | None, file_info: dict[str, str], upload_timestamp: int, account_id: str, bucket_id: str, action: str, content_md5: str | None, server_side_encryption: EncryptionSetting, file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, replication_status: ReplicationStatus | None = None, ): self.account_id = account_id self.bucket_id = bucket_id self.content_md5 = content_md5 self.action = action super().__init__( api=api, id_=id_, file_name=file_name, size=size, content_type=content_type, content_sha1=content_sha1, file_info=file_info, upload_timestamp=upload_timestamp, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, replication_status=replication_status, ) @property def cache_control(self) -> str | None: return self.file_info.get('b2-cache-control') @property def expires(self) -> str | None: return self.file_info.get('b2-expires') def expires_parsed(self) -> dt.datetime | None: """Return the expiration date as a datetime object, or None if there is no expiration date. Raise ValueError if `expires` property is not a valid HTTP-date.""" if self.expires is None: return None return parse_http_date(self.expires) @property def content_disposition(self) -> str | None: return self.file_info.get('b2-content-disposition') @property def content_encoding(self) -> str | None: return self.file_info.get('b2-content-encoding') @property def content_language(self) -> str | None: return self.file_info.get('b2-content-language') def _get_args_for_clone(self): args = super()._get_args_for_clone() args.update( { 'account_id': self.account_id, 'bucket_id': self.bucket_id, 'action': self.action, 'content_md5': self.content_md5, } ) return args def as_dict(self): result = super().as_dict() result['accountId'] = self.account_id result['bucketId'] = self.bucket_id if self.action is not None: result['action'] = self.action if self.content_md5 is not None: result['contentMd5'] = self.content_md5 return result def get_fresh_state(self) -> FileVersion: """ Fetch all the information about this file version and return a new FileVersion object. This method does NOT change the object it is called on. """ return self.api.get_file_info(self.id_) def download( self, progress_listener: AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: EncryptionSetting | None = None, ) -> DownloadedFile: return self.api.download_file_by_id( self.id_, progress_listener=progress_listener, range_=range_, encryption=encryption, ) def _get_upload_headers(self) -> bytes: """ Return encoded http headers, as when sending an upload request to b2 http api. WARNING: the headers do not contain newlines between headers and spaces between key and value. This implementation is in par with ADVANCED_HEADERS_LIMIT and is reasonable only for `has_large_header` method """ # sometimes secret is not available, but we want to calculate headers # size anyway; to bypass this, we use a fake encryption setting # with a fake key sse = self.server_side_encryption if sse and sse.key and sse.key.secret is None: sse = deepcopy(sse) sse.key.secret = b'*' * sse.algorithm.get_length() headers = self.api.raw_api.get_upload_file_headers( upload_auth_token=self.api.account_info.get_account_auth_token(), file_name=self.file_name, content_length=self.size, content_type=self.content_type, content_sha1=self.content_sha1, file_info=self.file_info, server_side_encryption=sse, file_retention=self.file_retention, legal_hold=self.legal_hold, ) headers_str = ''.join( f'{key}{value}' for key, value in headers.items() if value is not None ) return headers_str.encode('utf8') @property def has_large_header(self) -> bool: """ Determine whether FileVersion's info fits header size limit defined by B2. This function makes sense only for "advanced" buckets, i.e. those which have Server-Side Encryption or File Lock enabled. See https://www.backblaze.com/b2/docs/files.html#httpHeaderSizeLimit. """ return len(self._get_upload_headers()) > self.ADVANCED_HEADERS_LIMIT class DownloadVersion(BaseFileVersion): """ A structure which represents metadata of an initialized download """ __slots__ = [ 'range_', 'content_disposition', 'content_length', 'content_language', 'expires', 'cache_control', 'content_encoding', ] def __init__( self, api: B2Api, id_: str, file_name: str, size: int, content_type: str | None, content_sha1: str | None, file_info: dict[str, str], upload_timestamp: int, server_side_encryption: EncryptionSetting, range_: Range, content_disposition: str | None, content_length: int, content_language: str | None, expires: str | None, cache_control: str | None, content_encoding: str | None, file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, replication_status: ReplicationStatus | None = None, ): self.range_ = range_ self.content_disposition = content_disposition self.content_length = content_length self.content_language = content_language self.expires = expires self.cache_control = cache_control self.content_encoding = content_encoding super().__init__( api=api, id_=id_, file_name=file_name, size=size, content_type=content_type, content_sha1=content_sha1, file_info=file_info, upload_timestamp=upload_timestamp, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, replication_status=replication_status, ) def expires_parsed(self) -> dt.datetime | None: """Return the expiration date as a datetime object, or None if there is no expiration date. Raise ValueError if `expires` property is not a valid HTTP-date.""" if self.expires is None: return None return parse_http_date(self.expires) def as_dict(self) -> dict: result = super().as_dict() if self.cache_control is not None: result['cacheControl'] = self.cache_control if self.expires is not None: result['expires'] = self.expires if self.content_disposition is not None: result['contentDisposition'] = self.content_disposition if self.content_encoding is not None: result['contentEncoding'] = self.content_encoding if self.content_language is not None: result['contentLanguage'] = self.content_language return result def _get_args_for_clone(self): args = super()._get_args_for_clone() args.update( { 'range_': self.range_, 'content_disposition': self.content_disposition, 'content_length': self.content_length, 'content_language': self.content_language, 'expires': self.expires, 'cache_control': self.cache_control, 'content_encoding': self.content_encoding, } ) return args class FileVersionFactory: """ Construct :py:class:`b2sdk.v2.FileVersion` objects from api responses. """ FILE_VERSION_CLASS = FileVersion def __init__(self, api: B2Api): self.api = api def from_api_response(self, file_version_dict, force_action=None): """ Turn this: .. code-block:: python { "action": "hide", "fileId": "4_zBucketName_f103b7ca31313c69c_d20151230_m030117_c001_v0001015_t0000", "fileName": "randomdata", "size": 0, "uploadTimestamp": 1451444477000, "replicationStatus": "pending" } or this: .. code-block:: python { "accountId": "4aa9865d6f00", "bucketId": "547a2a395826655d561f0010", "contentLength": 1350, "contentSha1": "753ca1c2d0f3e8748320b38f5da057767029a036", "contentType": "application/octet-stream", "fileId": "4_z547a2a395826655d561f0010_f106d4ca95f8b5b78_d20160104_m003906_c001_v0001013_t0005", "fileInfo": {}, "fileName": "randomdata", "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"}, "replicationStatus": "completed" } into a :py:class:`b2sdk.v2.FileVersion` object. """ assert ( file_version_dict.get('action') is None or force_action is None ), 'action was provided by both info_dict and function argument' action = file_version_dict.get('action') or force_action file_name = file_version_dict['fileName'] id_ = file_version_dict['fileId'] if 'size' in file_version_dict: size = file_version_dict['size'] elif 'contentLength' in file_version_dict: size = file_version_dict['contentLength'] else: raise ValueError('no size or contentLength') upload_timestamp = file_version_dict.get('uploadTimestamp') content_type = file_version_dict.get('contentType') content_sha1 = file_version_dict.get('contentSha1') content_md5 = file_version_dict.get('contentMd5') file_info = file_version_dict.get('fileInfo') server_side_encryption = EncryptionSettingFactory.from_file_version_dict(file_version_dict) file_retention = FileRetentionSetting.from_file_version_dict(file_version_dict) legal_hold = LegalHold.from_file_version_dict(file_version_dict) replication_status_value = file_version_dict.get('replicationStatus') replication_status = ( replication_status_value and ReplicationStatus[replication_status_value.upper()] ) return self.FILE_VERSION_CLASS( self.api, id_, file_name, size, content_type, content_sha1, file_info, upload_timestamp, file_version_dict['accountId'], file_version_dict['bucketId'], action, content_md5, server_side_encryption, file_retention, legal_hold, replication_status, ) class DownloadVersionFactory: """ Construct :py:class:`b2sdk.v2.DownloadVersion` objects from download headers. """ def __init__(self, api: B2Api): self.api = api @classmethod def range_and_size_from_header(cls, header: str) -> tuple[Range, int]: range_, size = Range.from_header_with_size(header) assert size is not None, 'Total length was expected in Content-Range header' return range_, size @classmethod def file_info_from_headers(cls, headers: dict) -> dict: file_info = {} prefix_len = len(FILE_INFO_HEADER_PREFIX_LOWER) for header_name, header_value in headers.items(): if header_name[:prefix_len].lower() == FILE_INFO_HEADER_PREFIX_LOWER: file_info_key = header_name[prefix_len:] file_info[file_info_key] = b2_url_decode(header_value) return file_info def from_response_headers(self, headers): file_info = self.file_info_from_headers(headers) content_range_header_value = headers.get('Content-Range') if content_range_header_value: range_, size = self.range_and_size_from_header(content_range_header_value) content_length = int(headers['Content-Length']) else: size = content_length = int(headers['Content-Length']) range_ = Range(0, size - 1) if size else EMPTY_RANGE return DownloadVersion( api=self.api, id_=headers['x-bz-file-id'], file_name=b2_url_decode(headers['x-bz-file-name']), size=size, content_type=headers['content-type'], content_sha1=headers['x-bz-content-sha1'], file_info=file_info, upload_timestamp=int(headers['x-bz-upload-timestamp']), server_side_encryption=EncryptionSettingFactory.from_response_headers(headers), range_=range_, content_disposition=headers.get('Content-Disposition'), content_length=content_length, content_language=headers.get('Content-Language'), expires=headers.get('Expires'), cache_control=headers.get('Cache-Control'), content_encoding=headers.get('Content-Encoding'), file_retention=FileRetentionSetting.from_response_headers(headers), legal_hold=LegalHold.from_response_headers(headers), replication_status=ReplicationStatus.from_response_headers(headers), ) class FileIdAndName: """ A structure which represents a B2 cloud file with just `file_name` and `fileId` attributes. Used to return data from calls to b2_delete_file_version and b2_cancel_large_file. """ def __init__(self, file_id: str, file_name: str): self.file_id = file_id self.file_name = file_name @classmethod def from_cancel_or_delete_response(cls, response): return cls(response['fileId'], response['fileName']) def as_dict(self): """represents the object as a dict which looks almost exactly like the raw api output for delete_file_version""" return {'action': 'delete', 'fileId': self.file_id, 'fileName': self.file_name} def __eq__(self, other): return self.file_id == other.file_id and self.file_name == other.file_name def __repr__(self): return f'{self.__class__.__name__}({repr(self.file_id)}, {repr(self.file_name)})' b2-sdk-python-2.8.0/b2sdk/_internal/filter.py000066400000000000000000000037151474454370000207570ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/filter.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import fnmatch from dataclasses import dataclass from enum import Enum from typing import Sequence class FilterType(Enum): INCLUDE = 'include' EXCLUDE = 'exclude' @dataclass class Filter: type: FilterType pattern: str @classmethod def include(cls, pattern: str) -> Filter: return cls(type=FilterType.INCLUDE, pattern=pattern) @classmethod def exclude(cls, pattern: str) -> Filter: return cls(type=FilterType.EXCLUDE, pattern=pattern) class FilterMatcher: """ Holds a list of filters and matches a string (i.e. file name) against them. The order of filters matters. The *last* matching filter decides whether the string is included or excluded. If no filter matches, the string is included by default. If the given list of filters contains only INCLUDE filters, then it is assumed that all files are excluded by default. In this case, an additional EXCLUDE filter is prepended to the list. :param filters: list of filters """ def __init__(self, filters: Sequence[Filter]): if filters and all(filter_.type == FilterType.INCLUDE for filter_ in filters): filters = [Filter(type=FilterType.EXCLUDE, pattern='*'), *filters] self.filters = filters def match(self, s: str) -> bool: include_file = True for filter_ in self.filters: matched = fnmatch.fnmatchcase(s, filter_.pattern) if matched and filter_.type == FilterType.INCLUDE: include_file = True elif matched and filter_.type == FilterType.EXCLUDE: include_file = False return include_file b2-sdk-python-2.8.0/b2sdk/_internal/http_constants.py000066400000000000000000000030231474454370000225350ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/http_constants.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import string # These constants are needed in different modules, so they are stored in this module, that # imports nothing, thus avoiding circular imports # https://www.backblaze.com/docs/cloud-storage-buckets#bucket-names BUCKET_NAME_CHARS = string.ascii_lowercase + string.digits + '-' BUCKET_NAME_CHARS_UNIQ = string.ascii_lowercase + string.digits + '-' BUCKET_NAME_LENGTH_RANGE = (6, 63) LIST_FILE_NAMES_MAX_LIMIT = 10000 # https://www.backblaze.com/b2/docs/b2_list_file_names.html FILE_INFO_HEADER_PREFIX = 'X-Bz-Info-' FILE_INFO_HEADER_PREFIX_LOWER = FILE_INFO_HEADER_PREFIX.lower() # Standard names for file info entries SRC_LAST_MODIFIED_MILLIS = 'src_last_modified_millis' # SHA-1 hash key for large files LARGE_FILE_SHA1 = 'large_file_sha1' # Special X-Bz-Content-Sha1 value to verify checksum at the end HEX_DIGITS_AT_END = 'hex_digits_at_end' # Identifying SSE_C keys SSE_C_KEY_ID_FILE_INFO_KEY_NAME = 'sse_c_key_id' SSE_C_KEY_ID_HEADER = FILE_INFO_HEADER_PREFIX + SSE_C_KEY_ID_FILE_INFO_KEY_NAME # Default part sizes MEGABYTE = 1000 * 1000 GIGABYTE = 1000 * MEGABYTE DEFAULT_MIN_PART_SIZE = 5 * MEGABYTE DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE = 100 * MEGABYTE DEFAULT_MAX_PART_SIZE = 5 * GIGABYTE b2-sdk-python-2.8.0/b2sdk/_internal/included_sources.py000066400000000000000000000015371474454370000230240ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/included_sources.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # This module provides a list of third party sources included and modified in b2sdk, so it can be exposed to # B2 Command Line Tool for printing, for legal compliance reasons import dataclasses _included_sources: list[IncludedSourceMeta] = [] @dataclasses.dataclass class IncludedSourceMeta: name: str comment: str files: dict[str, str] def add_included_source(src: IncludedSourceMeta): _included_sources.append(src) def get_included_sources() -> list[IncludedSourceMeta]: return _included_sources b2-sdk-python-2.8.0/b2sdk/_internal/large_file/000077500000000000000000000000001474454370000212035ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/large_file/__init__.py000066400000000000000000000005241474454370000233150ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/large_file/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/large_file/part.py000066400000000000000000000026461474454370000225330ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/large_file/part.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations class PartFactory: @classmethod def from_list_parts_dict(cls, part_dict): return Part( part_dict['fileId'], part_dict['partNumber'], part_dict['contentLength'], part_dict['contentSha1'], ) class Part: """ A structure which represents a *part* of a large file upload. :ivar str ~.file_id: ``fileId`` :ivar int ~.part_number: part number, starting with 1 :ivar str ~.content_length: content length, in bytes :ivar str ~.content_sha1: checksum """ def __init__(self, file_id, part_number, content_length, content_sha1): self.file_id = file_id self.part_number = part_number self.content_length = content_length self.content_sha1 = content_sha1 def __repr__(self): return f'<{self.__class__.__name__} {self.file_id} {self.part_number} {self.content_length} {self.content_sha1}>' def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) b2-sdk-python-2.8.0/b2sdk/_internal/large_file/services.py000066400000000000000000000112501474454370000233770ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/large_file/services.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal.encryption.setting import EncryptionSetting from b2sdk._internal.file_lock import FileRetentionSetting, LegalHold from b2sdk._internal.file_version import FileIdAndName from b2sdk._internal.large_file.part import PartFactory from b2sdk._internal.large_file.unfinished_large_file import UnfinishedLargeFile class LargeFileServices: UNFINISHED_LARGE_FILE_CLASS = staticmethod(UnfinishedLargeFile) def __init__(self, services): self.services = services def list_parts(self, file_id, start_part_number=None, batch_size=None): """ Generator that yields a :py:class:`b2sdk.v2.Part` for each of the parts that have been uploaded. :param str file_id: the ID of the large file that is not finished :param int start_part_number: the first part number to return; defaults to the first part :param int batch_size: the number of parts to fetch at a time from the server :rtype: generator """ batch_size = batch_size or 100 while True: response = self.services.session.list_parts(file_id, start_part_number, batch_size) for part_dict in response['parts']: yield PartFactory.from_list_parts_dict(part_dict) start_part_number = response.get('nextPartNumber') if start_part_number is None: break def list_unfinished_large_files( self, bucket_id, start_file_id=None, batch_size=None, prefix=None ): """ A generator that yields an :py:class:`b2sdk.v2.UnfinishedLargeFile` for each unfinished large file in the bucket, starting at the given file, filtering by prefix. :param str bucket_id: bucket id :param str,None start_file_id: a file ID to start from or None to start from the beginning :param int,None batch_size: max file count :param str,None prefix: file name prefix filter :rtype: generator[b2sdk.v2.UnfinishedLargeFile] """ batch_size = batch_size or 100 while True: batch = self.services.session.list_unfinished_large_files( bucket_id, start_file_id, batch_size, prefix ) for file_dict in batch['files']: yield self.UNFINISHED_LARGE_FILE_CLASS(file_dict) start_file_id = batch.get('nextFileId') if start_file_id is None: break def get_unfinished_large_file(self, bucket_id, large_file_id, prefix=None): result = list( self.list_unfinished_large_files( bucket_id, start_file_id=large_file_id, batch_size=1, prefix=prefix ) ) if not result: return None unfinished_large_file = result[0] if unfinished_large_file.file_id != large_file_id: return None return unfinished_large_file def start_large_file( self, bucket_id, file_name, content_type=None, file_info=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, ): """ Start a large file transfer. :param str file_name: a file name :param str,None content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param dict,None file_info: a file info to store with the file or ``None`` to not store anything :param b2sdk.v2.EncryptionSetting encryption: encryption settings (``None`` if unknown) :param b2sdk.v2.FileRetentionSetting file_retention: file retention setting :param b2sdk.v2.LegalHold legal_hold: legal hold setting """ return self.UNFINISHED_LARGE_FILE_CLASS( self.services.session.start_large_file( bucket_id, file_name, content_type, file_info, server_side_encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) ) # delete/cancel def cancel_large_file(self, file_id: str) -> FileIdAndName: """ Cancel a large file upload. """ response = self.services.session.cancel_large_file(file_id) return FileIdAndName.from_cancel_or_delete_response(response) b2-sdk-python-2.8.0/b2sdk/_internal/large_file/unfinished_large_file.py000066400000000000000000000055531474454370000260720ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/large_file/unfinished_large_file.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import datetime as dt from b2sdk._internal.encryption.setting import EncryptionSettingFactory from b2sdk._internal.file_lock import FileRetentionSetting, LegalHold from b2sdk._internal.utils.http_date import parse_http_date class UnfinishedLargeFile: """ A structure which represents a version of a file (in B2 cloud). :ivar str ~.file_id: ``fileId`` :ivar str ~.file_name: full file name (with path) :ivar str ~.account_id: account ID :ivar str ~.bucket_id: bucket ID :ivar str ~.content_type: :rfc:`822` content type, for example ``"application/octet-stream"`` :ivar dict ~.file_info: file info dict """ def __init__(self, file_dict): """ Initialize from one file returned by ``b2_start_large_file`` or ``b2_list_unfinished_large_files``. """ self.file_id = file_dict['fileId'] self.file_name = file_dict['fileName'] self.account_id = file_dict['accountId'] self.bucket_id = file_dict['bucketId'] self.content_type = file_dict['contentType'] self.file_info = file_dict['fileInfo'] self.encryption = EncryptionSettingFactory.from_file_version_dict(file_dict) self.file_retention = FileRetentionSetting.from_file_version_dict(file_dict) self.legal_hold = LegalHold.from_file_version_dict(file_dict) @property def cache_control(self) -> str | None: return (self.file_info or {}).get('b2-cache-control') @property def expires(self) -> str | None: return (self.file_info or {}).get('b2-expires') def expires_parsed(self) -> dt.datetime | None: """Return the expiration date as a datetime object, or None if there is no expiration date. Raise ValueError if `expires` property is not a valid HTTP-date.""" if self.expires is None: return None return parse_http_date(self.expires) @property def content_disposition(self) -> str | None: return (self.file_info or {}).get('b2-content-disposition') @property def content_encoding(self) -> str | None: return (self.file_info or {}).get('b2-content-encoding') @property def content_language(self) -> str | None: return (self.file_info or {}).get('b2-content-language') def __repr__(self): return f'<{self.__class__.__name__} {self.bucket_id} {self.file_name}>' def __eq__(self, other): return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) b2-sdk-python-2.8.0/b2sdk/_internal/progress.py000066400000000000000000000157641474454370000213450ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/progress.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import time from abc import ABCMeta, abstractmethod from .utils.escape import escape_control_chars try: from tqdm import tqdm # displays a nice progress bar except ImportError: tqdm = None # noqa class AbstractProgressListener(metaclass=ABCMeta): """ Interface expected by B2Api upload and download methods to report on progress. This interface just accepts the number of bytes transferred so far. Subclasses will need to know the total size if they want to report a percent done. """ def __init__(self, description: str = ''): self.description = description self._closed = False @abstractmethod def set_total_bytes(self, total_byte_count: int) -> None: """ Always called before __enter__ to set the expected total number of bytes. May be called more than once if an upload is retried. :param total_byte_count: expected total number of bytes """ @abstractmethod def bytes_completed(self, byte_count: int) -> None: """ Report the given number of bytes that have been transferred so far. This is not a delta, it is the total number of bytes transferred so far. Transfer can fail and restart from the beginning, so byte count can decrease between calls. :param byte_count: number of bytes have been transferred """ def _can_change_description(self) -> bool: """ Determines, on a per-implementation basis, whether the description can be changed at this time. """ return True def change_description(self, new_description: str) -> bool: """ Ability to change the description after the listener is started. Note: whether the change of description is allowed depends on the implementation. The safest option is to change the description before setting the total bytes. :param new_description: the new description to be used :return: information whether the description was changed """ if not self._can_change_description(): return False self.description = new_description return True def close(self) -> None: """ Must be called when you're done with the listener. In well-structured code, should be called only once. """ # import traceback, sys; traceback.print_stack(file=sys.stdout) assert ( self._closed is False ), 'progress listener was closed twice! uncomment the line above to debug this' self._closed = True def __enter__(self): """ A standard context manager method. """ return self def __exit__(self, exc_type, exc_val, exc_tb): """ A standard context manager method. """ self.close() class TqdmProgressListener(AbstractProgressListener): """ Progress listener based on tqdm library. This listener displays a nice progress bar, but requires `tqdm` package to be installed. """ def __init__(self, *args, **kwargs): if tqdm is None: raise ModuleNotFoundError("No module named 'tqdm' found") self.tqdm = None # set in set_total_bytes() self.prev_value = 0 super().__init__(*args, **kwargs) def set_total_bytes(self, total_byte_count: int) -> None: if self.tqdm is None: self.tqdm = tqdm( desc=escape_control_chars(self.description), total=total_byte_count, unit='B', unit_scale=True, leave=True, miniters=1, smoothing=0.1, mininterval=0.2, ) def bytes_completed(self, byte_count: int) -> None: # tqdm doesn't support running the progress bar backwards, # so on an upload retry, it just won't move until it gets # past the point where it failed. if self.prev_value < byte_count: self.tqdm.update(byte_count - self.prev_value) self.prev_value = byte_count def _can_change_description(self) -> bool: return self.tqdm is None def close(self) -> None: if self.tqdm is not None: self.tqdm.close() super().close() class SimpleProgressListener(AbstractProgressListener): """ Just a simple progress listener which prints info on a console. """ def __init__(self, *args, **kwargs): self.complete = 0 self.last_time = time.time() self.any_printed = False self.total = 0 # set in set_total_bytes() super().__init__(*args, **kwargs) def set_total_bytes(self, total_byte_count: int) -> None: self.total = total_byte_count def bytes_completed(self, byte_count: int) -> None: now = time.time() elapsed = now - self.last_time if 3 <= elapsed and self.total != 0: if not self.any_printed: print(escape_control_chars(self.description)) print(' %d%%' % int(100.0 * byte_count / self.total)) self.last_time = now self.any_printed = True def _can_change_description(self) -> bool: return not self.any_printed def close(self) -> None: if self.any_printed: print(' DONE.') super().close() class DoNothingProgressListener(AbstractProgressListener): """ This listener gives no output whatsoever. """ def set_total_bytes(self, total_byte_count: int) -> None: pass def bytes_completed(self, byte_count: int) -> None: pass class ProgressListenerForTest(AbstractProgressListener): """ Capture all the calls so they can be checked. """ def __init__(self, *args, **kwargs): self.calls = [] super().__init__(*args, **kwargs) def set_total_bytes(self, total_byte_count: int) -> None: self.calls.append('set_total_bytes(%d)' % (total_byte_count,)) def bytes_completed(self, byte_count: int) -> None: self.calls.append('bytes_completed(%d)' % (byte_count,)) def close(self) -> None: self.calls.append('close()') super().close() def get_calls(self) -> list[str]: return self.calls def make_progress_listener(description: str, quiet: bool) -> AbstractProgressListener: """ Produce the best progress listener available for the given parameters. :param description: listener description :param quiet: if ``True``, do not output anything :return: a listener object """ if quiet: return DoNothingProgressListener() elif tqdm is not None: return TqdmProgressListener(description) else: return SimpleProgressListener(description) b2-sdk-python-2.8.0/b2sdk/_internal/raw_api.py000066400000000000000000001155571474454370000211240ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/raw_api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import base64 from abc import ABCMeta, abstractmethod from enum import Enum, unique from logging import getLogger from typing import Any, Iterable from .utils.escape import unprintable_to_hex from .utils.typing import JSON try: from typing_extensions import Literal, NotRequired, TypedDict except ImportError: from typing import Literal, NotRequired, TypedDict from .encryption.setting import EncryptionMode, EncryptionSetting from .exception import ( AccessDenied, FileOrBucketNotFound, InvalidMetadataDirective, ResourceNotFound, RetentionWriteError, SSECKeyError, UnusableFileName, WrongEncryptionModeForBucketDefault, ) from .file_lock import BucketRetentionSetting, FileRetentionSetting, LegalHold from .http_constants import FILE_INFO_HEADER_PREFIX from .replication.setting import ReplicationConfiguration from .types import NotRequired, PositiveInt, TypedDict from .utils import b2_url_encode from .utils.docs import ensure_b2sdk_doc_urls # All supported realms REALM_URLS = { 'production': 'https://api.backblazeb2.com', 'dev': 'http://api.backblazeb2.xyz:8180', 'staging': 'https://api.backblaze.net', } # All possible capabilities ALL_CAPABILITIES = [ 'listKeys', 'writeKeys', 'deleteKeys', 'listBuckets', 'listAllBucketNames', 'readBuckets', 'writeBuckets', 'deleteBuckets', 'readBucketEncryption', 'writeBucketEncryption', 'readBucketRetentions', 'writeBucketRetentions', 'readFileRetentions', 'writeFileRetentions', 'readFileLegalHolds', 'writeFileLegalHolds', 'readBucketReplications', 'writeBucketReplications', 'bypassGovernance', 'listFiles', 'readFiles', 'shareFiles', 'writeFiles', 'deleteFiles', 'readBucketNotifications', 'writeBucketNotifications', ] # API version number to use when calling the service API_VERSION = 'v3' logger = getLogger(__name__) @unique class MetadataDirectiveMode(Enum): """Mode of handling metadata when copying a file""" COPY = 401 #: copy metadata from the source file REPLACE = 402 #: ignore the source file metadata and set it to provided values @ensure_b2sdk_doc_urls class LifecycleRule(TypedDict): """ Lifecycle Rule. External documentation: `B2 Cloud Storage Lifecycle Rules`_. .. _B2 Cloud Storage Lifecycle Rules: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules """ fileNamePrefix: str daysFromHidingToDeleting: NotRequired[PositiveInt | None] daysFromUploadingToHiding: NotRequired[PositiveInt | None] daysFromStartingToCancelingUnfinishedLargeFiles: NotRequired[PositiveInt | None] class NameValueDict(TypedDict): name: str value: str class NotificationTargetConfiguration(TypedDict): """ Notification Target Configuration. `hmacSha256SigningSecret`, if present, has to be a string of 32 alphanumeric characters. """ # TODO: add URL to the documentation targetType: Literal['webhook'] url: str customHeaders: NotRequired[list[NameValueDict] | None] hmacSha256SigningSecret: NotRequired[str | None] EVENT_TYPE = Literal[ 'b2:ObjectCreated:*', 'b2:ObjectCreated:Upload', 'b2:ObjectCreated:MultipartUpload', 'b2:ObjectCreated:Copy', 'b2:ObjectCreated:Replica', 'b2:ObjectCreated:MultipartReplica', 'b2:ObjectDeleted:*', 'b2:ObjectDeleted:Delete', 'b2:ObjectDeleted:LifecycleRule', 'b2:HideMarkerCreated:*', 'b2:HideMarkerCreated:Hide', 'b2:HideMarkerCreated:LifecycleRule', ] class _NotificationRule(TypedDict): """ Notification Rule. """ eventTypes: list[EVENT_TYPE] isEnabled: bool name: str objectNamePrefix: str targetConfiguration: NotificationTargetConfiguration suspensionReason: NotRequired[str] class NotificationRule(_NotificationRule): """ Notification Rule. When creating or modifying a notification rule, `isSuspended` and `suspensionReason` are ignored. """ isSuspended: NotRequired[bool] class NotificationRuleResponse(_NotificationRule): isSuspended: bool def notification_rule_response_to_request(rule: NotificationRuleResponse) -> NotificationRule: """ Convert NotificationRuleResponse to NotificationRule. """ rule = rule.copy() for key in ('isSuspended', 'suspensionReason'): rule.pop(key, None) return rule class AbstractRawApi(metaclass=ABCMeta): """ Direct access to the B2 web apis. """ @abstractmethod def authorize_account(self, realm_url, application_key_id, application_key): pass @abstractmethod def cancel_large_file(self, api_url, account_auth_token, file_id): pass @abstractmethod def copy_file( self, api_url, account_auth_token, source_file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_bucket_id=None, destination_server_side_encryption: EncryptionSetting | None = None, source_server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, ): pass @abstractmethod def copy_part( self, api_url, account_auth_token, source_file_id, large_file_id, part_number, bytes_range=None, destination_server_side_encryption: EncryptionSetting | None = None, source_server_side_encryption: EncryptionSetting | None = None, ): pass @abstractmethod def create_bucket( self, api_url, account_auth_token, account_id, bucket_name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, default_server_side_encryption: EncryptionSetting | None = None, is_file_lock_enabled: bool | None = None, replication: ReplicationConfiguration | None = None, ): pass @abstractmethod def create_key( self, api_url, account_auth_token, account_id, capabilities, key_name, valid_duration_seconds, bucket_id, name_prefix, ): pass @abstractmethod def download_file_from_url( self, account_auth_token_or_none: str | None, url: str, range_: tuple[int, int] | None = None, encryption: EncryptionSetting | None = None, ): pass @abstractmethod def delete_key(self, api_url, account_auth_token, application_key_id): pass @abstractmethod def delete_bucket(self, api_url, account_auth_token, account_id, bucket_id): pass @abstractmethod def delete_file_version( self, api_url, account_auth_token, file_id, file_name, bypass_governance: bool = False ): pass @abstractmethod def finish_large_file(self, api_url, account_auth_token, file_id, part_sha1_array): pass @abstractmethod def get_download_authorization( self, api_url, account_auth_token, bucket_id, file_name_prefix, valid_duration_in_seconds ): pass @abstractmethod def get_file_info_by_id( self, api_url: str, account_auth_token: str, file_id: str ) -> dict[str, Any]: pass @abstractmethod def get_file_info_by_name( self, download_url: str, account_auth_token: str, bucket_name: str, file_name: str ) -> dict[str, Any]: pass @abstractmethod def get_upload_url(self, api_url, account_auth_token, bucket_id): pass @abstractmethod def get_upload_part_url(self, api_url, account_auth_token, file_id): pass @abstractmethod def hide_file(self, api_url, account_auth_token, bucket_id, file_name): pass @abstractmethod def list_buckets( self, api_url, account_auth_token, account_id, bucket_id=None, bucket_name=None, ): pass @abstractmethod def list_file_names( self, api_url, account_auth_token, bucket_id, start_file_name=None, max_file_count=None, prefix=None, ): pass @abstractmethod def list_file_versions( self, api_url, account_auth_token, bucket_id, start_file_name=None, start_file_id=None, max_file_count=None, prefix=None, ): pass @abstractmethod def list_keys( self, api_url, account_auth_token, account_id, max_key_count=None, start_application_key_id=None, ): pass @abstractmethod def list_parts(self, api_url, account_auth_token, file_id, start_part_number, max_part_count): pass @abstractmethod def list_unfinished_large_files( self, api_url, account_auth_token, bucket_id, start_file_id=None, max_file_count=None, prefix=None, ): pass @abstractmethod def start_large_file( self, api_url, account_auth_token, bucket_id, file_name, content_type, file_info, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, ): pass @abstractmethod def update_bucket( self, api_url, account_auth_token, account_id, bucket_id, bucket_type=None, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, if_revision_is=None, default_server_side_encryption: EncryptionSetting | None = None, default_retention: BucketRetentionSetting | None = None, replication: ReplicationConfiguration | None = None, is_file_lock_enabled: bool | None = None, ): pass @abstractmethod def update_file_retention( self, api_url, account_auth_token, file_id, file_name, file_retention: FileRetentionSetting, bypass_governance: bool = False, ): pass @classmethod def get_upload_file_headers( cls, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_info: dict, server_side_encryption: EncryptionSetting | None, file_retention: FileRetentionSetting | None, legal_hold: LegalHold | None, custom_upload_timestamp: int | None = None, ) -> dict: headers = { 'Authorization': upload_auth_token, 'Content-Length': str(content_length), 'X-Bz-File-Name': b2_url_encode(file_name), 'Content-Type': content_type, 'X-Bz-Content-Sha1': content_sha1, } for k, v in file_info.items(): headers[FILE_INFO_HEADER_PREFIX + k] = b2_url_encode(v) if server_side_encryption is not None: assert server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C, ) server_side_encryption.add_to_upload_headers(headers) if legal_hold is not None: legal_hold.add_to_upload_headers(headers) if file_retention is not None: file_retention.add_to_to_upload_headers(headers) if custom_upload_timestamp is not None: headers['X-Bz-Custom-Upload-Timestamp'] = str(custom_upload_timestamp) return headers @abstractmethod def upload_file( self, upload_url, upload_auth_token, file_name, content_length, content_type, content_sha1, file_info, data_stream, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): pass @abstractmethod def upload_part( self, upload_url, upload_auth_token, part_number, content_length, sha1_sum, input_stream, server_side_encryption: EncryptionSetting | None = None, ): pass def get_download_url_by_id(self, download_url, file_id): return f'{download_url}/b2api/{API_VERSION}/b2_download_file_by_id?fileId={file_id}' def get_download_url_by_name(self, download_url, bucket_name, file_name): return download_url + '/file/' + bucket_name + '/' + b2_url_encode(file_name) @abstractmethod def set_bucket_notification_rules( self, api_url: str, account_auth_token: str, bucket_id: str, rules: Iterable[NotificationRule], ) -> list[NotificationRuleResponse]: pass @abstractmethod def get_bucket_notification_rules( self, api_url: str, account_auth_token: str, bucket_id: str ) -> list[NotificationRuleResponse]: pass class B2RawHTTPApi(AbstractRawApi): """ Provide access to the B2 web APIs, exactly as they are provided by b2. Requires that you provide all necessary URLs and auth tokens for each call. Each API call decodes the returned JSON and returns a dict. For details on what each method does, see the B2 docs: https://www.backblaze.com/b2/docs/ This class is intended to be a super-simple, very thin layer on top of the HTTP calls. It can be mocked-out for testing higher layers. And this class can be tested by exercising each call just once, which is relatively quick. """ def __init__(self, b2_http): self.b2_http = b2_http def _post_json(self, base_url: str, endpoint: str, auth: str, **params) -> JSON: """ A helper method for calling an API with the given auth and params. :param base_url: something like "https://api001.backblazeb2.com/" :param auth: passed in Authorization header :param endpoint: example: "b2_create_bucket" :param args: the rest of the parameters are passed to b2 :return: the decoded JSON response """ url = f'{base_url}/b2api/{API_VERSION}/{endpoint}' headers = {'Authorization': auth} return self.b2_http.post_json_return_json(url, headers, params) def _get_json(self, base_url: str, endpoint: str, auth: str, **params) -> JSON: url = f'{base_url}/b2api/{API_VERSION}/{endpoint}' headers = {'Authorization': auth} return self.b2_http.request_content_return_json('GET', url, headers, params=params) def authorize_account(self, realm_url, application_key_id, application_key): auth = ( f"Basic {base64.b64encode(f'{application_key_id}:{application_key}'.encode()).decode()}" ) return self._post_json(realm_url, 'b2_authorize_account', auth) def cancel_large_file(self, api_url, account_auth_token, file_id): return self._post_json(api_url, 'b2_cancel_large_file', account_auth_token, fileId=file_id) def create_bucket( self, api_url, account_auth_token, account_id, bucket_name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, default_server_side_encryption: EncryptionSetting | None = None, is_file_lock_enabled: bool | None = None, replication: ReplicationConfiguration | None = None, ): kwargs = dict( accountId=account_id, bucketName=bucket_name, bucketType=bucket_type, ) if bucket_info is not None: kwargs['bucketInfo'] = bucket_info if cors_rules is not None: kwargs['corsRules'] = cors_rules if lifecycle_rules is not None: kwargs['lifecycleRules'] = lifecycle_rules if default_server_side_encryption is not None: if not default_server_side_encryption.mode.can_be_set_as_bucket_default(): raise WrongEncryptionModeForBucketDefault(default_server_side_encryption.mode) kwargs['defaultServerSideEncryption'] = ( default_server_side_encryption.serialize_to_json_for_request() ) if is_file_lock_enabled is not None: kwargs['fileLockEnabled'] = is_file_lock_enabled if replication is not None: kwargs['replicationConfiguration'] = replication.serialize_to_json_for_request() return self._post_json( api_url, 'b2_create_bucket', account_auth_token, **kwargs, ) def create_key( self, api_url, account_auth_token, account_id, capabilities, key_name, valid_duration_seconds, bucket_id, name_prefix, ): return self._post_json( api_url, 'b2_create_key', account_auth_token, accountId=account_id, capabilities=capabilities, keyName=key_name, validDurationInSeconds=valid_duration_seconds, bucketId=bucket_id, namePrefix=name_prefix, ) def delete_bucket(self, api_url, account_auth_token, account_id, bucket_id): return self._post_json( api_url, 'b2_delete_bucket', account_auth_token, accountId=account_id, bucketId=bucket_id, ) def delete_file_version( self, api_url, account_auth_token, file_id, file_name, bypass_governance: bool = False ): return self._post_json( api_url, 'b2_delete_file_version', account_auth_token, fileId=file_id, fileName=file_name, bypassGovernance=bypass_governance, ) def delete_key(self, api_url, account_auth_token, application_key_id): return self._post_json( api_url, 'b2_delete_key', account_auth_token, applicationKeyId=application_key_id, ) def download_file_from_url( self, account_auth_token_or_none: str | None, url: str, range_: tuple[int, int] | None = None, encryption: EncryptionSetting | None = None, ): """ Issue a streaming request for download of a file, potentially authorized. :param account_auth_token_or_none: an optional account auth token to pass in :param url: the full URL to download from :param range_: two-element tuple for http Range header :param b2sdk.v2.EncryptionSetting encryption: encryption settings for downloading :return: b2_http response """ request_headers = {} _add_range_header(request_headers, range_) if encryption is not None: assert encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C, ) encryption.add_to_download_headers(request_headers) if account_auth_token_or_none is not None: request_headers['Authorization'] = account_auth_token_or_none try: return self.b2_http.get_content(url, request_headers) except AccessDenied: raise SSECKeyError() def finish_large_file(self, api_url, account_auth_token, file_id, part_sha1_array): return self._post_json( api_url, 'b2_finish_large_file', account_auth_token, fileId=file_id, partSha1Array=part_sha1_array, ) def get_download_authorization( self, api_url, account_auth_token, bucket_id, file_name_prefix, valid_duration_in_seconds ): return self._post_json( api_url, 'b2_get_download_authorization', account_auth_token, bucketId=bucket_id, fileNamePrefix=file_name_prefix, validDurationInSeconds=valid_duration_in_seconds, ) def get_file_info_by_id( self, api_url: str, account_auth_token: str, file_id: str ) -> dict[str, Any]: return self._post_json(api_url, 'b2_get_file_info', account_auth_token, fileId=file_id) def get_file_info_by_name( self, download_url: str, account_auth_token: str, bucket_name: str, file_name: str ) -> dict[str, Any]: download_url = self.get_download_url_by_name(download_url, bucket_name, file_name) try: response = self.b2_http.head_content( download_url, headers={'Authorization': account_auth_token} ) return response.headers except ResourceNotFound: logger.debug('Resource Not Found: %s' % download_url) raise FileOrBucketNotFound(bucket_name, file_name) def get_upload_url(self, api_url, account_auth_token, bucket_id): return self._post_json(api_url, 'b2_get_upload_url', account_auth_token, bucketId=bucket_id) def get_upload_part_url(self, api_url, account_auth_token, file_id): return self._post_json( api_url, 'b2_get_upload_part_url', account_auth_token, fileId=file_id ) def hide_file(self, api_url, account_auth_token, bucket_id, file_name): return self._post_json( api_url, 'b2_hide_file', account_auth_token, bucketId=bucket_id, fileName=file_name ) def list_buckets( self, api_url, account_auth_token, account_id, bucket_id=None, bucket_name=None, ): return self._post_json( api_url, 'b2_list_buckets', account_auth_token, accountId=account_id, bucketTypes=['all'], bucketId=bucket_id, bucketName=bucket_name, ) def list_file_names( self, api_url, account_auth_token, bucket_id, start_file_name=None, max_file_count=None, prefix=None, ): return self._post_json( api_url, 'b2_list_file_names', account_auth_token, bucketId=bucket_id, startFileName=start_file_name, maxFileCount=max_file_count, prefix=prefix, ) def list_file_versions( self, api_url, account_auth_token, bucket_id, start_file_name=None, start_file_id=None, max_file_count=None, prefix=None, ): return self._post_json( api_url, 'b2_list_file_versions', account_auth_token, bucketId=bucket_id, startFileName=start_file_name, startFileId=start_file_id, maxFileCount=max_file_count, prefix=prefix, ) def list_keys( self, api_url, account_auth_token, account_id, max_key_count=None, start_application_key_id=None, ): return self._post_json( api_url, 'b2_list_keys', account_auth_token, accountId=account_id, maxKeyCount=max_key_count, startApplicationKeyId=start_application_key_id, ) def list_parts(self, api_url, account_auth_token, file_id, start_part_number, max_part_count): return self._post_json( api_url, 'b2_list_parts', account_auth_token, fileId=file_id, startPartNumber=start_part_number, maxPartCount=max_part_count, ) def list_unfinished_large_files( self, api_url, account_auth_token, bucket_id, start_file_id=None, max_file_count=None, prefix=None, ): return self._post_json( api_url, 'b2_list_unfinished_large_files', account_auth_token, bucketId=bucket_id, startFileId=start_file_id, maxFileCount=max_file_count, namePrefix=prefix, ) def start_large_file( self, api_url, account_auth_token, bucket_id, file_name, content_type, file_info, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): kwargs = {} if server_side_encryption is not None: assert server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C, ) kwargs['serverSideEncryption'] = server_side_encryption.serialize_to_json_for_request() if server_side_encryption.mode == EncryptionMode.SSE_C: file_info = server_side_encryption.add_key_id_to_file_info(file_info) if legal_hold is not None: kwargs['legalHold'] = legal_hold.to_server() if file_retention is not None: kwargs['fileRetention'] = file_retention.serialize_to_json_for_request() if custom_upload_timestamp is not None: kwargs['custom_upload_timestamp'] = custom_upload_timestamp return self._post_json( api_url, 'b2_start_large_file', account_auth_token, bucketId=bucket_id, fileName=file_name, fileInfo=file_info, contentType=content_type, **kwargs, ) def update_bucket( self, api_url, account_auth_token, account_id, bucket_id, bucket_type=None, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, if_revision_is=None, default_server_side_encryption: EncryptionSetting | None = None, default_retention: BucketRetentionSetting | None = None, replication: ReplicationConfiguration | None = None, is_file_lock_enabled: bool | None = None, ): kwargs = {} if if_revision_is is not None: kwargs['ifRevisionIs'] = if_revision_is if bucket_info is not None: kwargs['bucketInfo'] = bucket_info if bucket_type is not None: kwargs['bucketType'] = bucket_type if cors_rules is not None: kwargs['corsRules'] = cors_rules if lifecycle_rules is not None: kwargs['lifecycleRules'] = lifecycle_rules if default_server_side_encryption is not None: if not default_server_side_encryption.mode.can_be_set_as_bucket_default(): raise WrongEncryptionModeForBucketDefault(default_server_side_encryption.mode) kwargs['defaultServerSideEncryption'] = ( default_server_side_encryption.serialize_to_json_for_request() ) if default_retention is not None: kwargs['defaultRetention'] = default_retention.serialize_to_json_for_request() if replication is not None: kwargs['replicationConfiguration'] = replication.serialize_to_json_for_request() if is_file_lock_enabled is not None: kwargs['fileLockEnabled'] = is_file_lock_enabled assert kwargs return self._post_json( api_url, 'b2_update_bucket', account_auth_token, accountId=account_id, bucketId=bucket_id, **kwargs, ) def update_file_retention( self, api_url, account_auth_token, file_id, file_name, file_retention: FileRetentionSetting, bypass_governance: bool = False, ): kwargs = {} kwargs['fileRetention'] = file_retention.serialize_to_json_for_request() try: return self._post_json( api_url, 'b2_update_file_retention', account_auth_token, fileId=file_id, fileName=file_name, bypassGovernance=bypass_governance, **kwargs, ) except AccessDenied: raise RetentionWriteError() def update_file_legal_hold( self, api_url, account_auth_token, file_id, file_name, legal_hold: LegalHold, ): try: return self._post_json( api_url, 'b2_update_file_legal_hold', account_auth_token, fileId=file_id, fileName=file_name, legalHold=legal_hold.to_server(), ) except AccessDenied: raise RetentionWriteError() def check_b2_filename(self, filename): """ Raise an appropriate exception with details if the filename is unusable. See https://www.backblaze.com/b2/docs/files.html for the rules. :param filename: a proposed filename in unicode :return: None if the filename is usable """ encoded_name = filename.encode('utf-8') length_in_bytes = len(encoded_name) if length_in_bytes < 1: raise UnusableFileName('Filename must be at least 1 character.') if length_in_bytes > 1024: raise UnusableFileName('Filename is too long (can be at most 1024 bytes).') lowest_unicode_value = ord(min(filename)) if lowest_unicode_value < 32: message = f'Filename "{unprintable_to_hex(filename)}" contains code {lowest_unicode_value} (hex {lowest_unicode_value:02x}), less than 32.' raise UnusableFileName(message) # No DEL for you. if '\x7f' in filename: raise UnusableFileName('DEL character (0x7f) not allowed.') if filename[0] == '/' or filename[-1] == '/': raise UnusableFileName("Filename may not start or end with '/'.") if '//' in filename: raise UnusableFileName('Filename may not contain "//".') long_segment = max([len(segment.encode('utf-8')) for segment in filename.split('/')]) if long_segment > 250: raise UnusableFileName('Filename segment too long (maximum 250 bytes in utf-8).') def upload_file( self, upload_url, upload_auth_token, file_name, content_length, content_type, content_sha1, file_info: dict, data_stream, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): """ Upload one, small file to b2. :param upload_url: the upload_url from b2_authorize_account :param upload_auth_token: the auth token from b2_authorize_account :param file_name: the name of the B2 file :param content_length: number of bytes in the file :param content_type: MIME type :param content_sha1: hex SHA1 of the contents of the file :param file_info: extra file info to upload :param data_stream: a file like object from which the contents of the file can be read :param server_side_encryption: encryption setting for the file :param file_retention: retention setting for the file :param legal_hold: legal hold setting for the file :param custom_upload_timestamp: custom upload timestamp for the file :return: """ # Raise UnusableFileName if the file_name doesn't meet the rules. self.check_b2_filename(file_name) headers = self.get_upload_file_headers( upload_auth_token=upload_auth_token, file_name=file_name, content_length=content_length, content_type=content_type, content_sha1=content_sha1, file_info=file_info, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) return self.b2_http.post_content_return_json(upload_url, headers, data_stream) def upload_part( self, upload_url, upload_auth_token, part_number, content_length, content_sha1, data_stream, server_side_encryption: EncryptionSetting | None = None, ): headers = { 'Authorization': upload_auth_token, 'Content-Length': str(content_length), 'X-Bz-Part-Number': str(part_number), 'X-Bz-Content-Sha1': content_sha1, } if server_side_encryption is not None: assert server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C, ) server_side_encryption.add_to_upload_headers(headers) return self.b2_http.post_content_return_json(upload_url, headers, data_stream) def copy_file( self, api_url, account_auth_token, source_file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_bucket_id=None, destination_server_side_encryption: EncryptionSetting | None = None, source_server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, ): kwargs = {} if bytes_range is not None: range_dict = {} _add_range_header(range_dict, bytes_range) kwargs['range'] = range_dict['Range'] if metadata_directive is not None: assert metadata_directive in tuple(MetadataDirectiveMode) if metadata_directive is MetadataDirectiveMode.COPY and ( content_type is not None or file_info is not None ): raise InvalidMetadataDirective( 'content_type and file_info should be None when metadata_directive is COPY' ) elif metadata_directive is MetadataDirectiveMode.REPLACE and content_type is None: raise InvalidMetadataDirective( 'content_type cannot be None when metadata_directive is REPLACE' ) kwargs['metadataDirective'] = metadata_directive.name if content_type is not None: kwargs['contentType'] = content_type if file_info is not None: kwargs['fileInfo'] = file_info if destination_bucket_id is not None: kwargs['destinationBucketId'] = destination_bucket_id if destination_server_side_encryption is not None: assert destination_server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C, ) kwargs['destinationServerSideEncryption'] = ( destination_server_side_encryption.serialize_to_json_for_request() ) if source_server_side_encryption is not None: assert source_server_side_encryption.mode == EncryptionMode.SSE_C kwargs['sourceServerSideEncryption'] = ( source_server_side_encryption.serialize_to_json_for_request() ) if legal_hold is not None: kwargs['legalHold'] = legal_hold.to_server() if file_retention is not None: kwargs['fileRetention'] = file_retention.serialize_to_json_for_request() try: return self._post_json( api_url, 'b2_copy_file', account_auth_token, sourceFileId=source_file_id, fileName=new_file_name, **kwargs, ) except AccessDenied: raise SSECKeyError() def copy_part( self, api_url, account_auth_token, source_file_id, large_file_id, part_number, bytes_range=None, destination_server_side_encryption: EncryptionSetting | None = None, source_server_side_encryption: EncryptionSetting | None = None, ): kwargs = {} if bytes_range is not None: range_dict = {} _add_range_header(range_dict, bytes_range) kwargs['range'] = range_dict['Range'] if destination_server_side_encryption is not None: assert destination_server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C, ) kwargs['destinationServerSideEncryption'] = ( destination_server_side_encryption.serialize_to_json_for_request() ) if source_server_side_encryption is not None: assert source_server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C, ) kwargs['sourceServerSideEncryption'] = ( source_server_side_encryption.serialize_to_json_for_request() ) try: return self._post_json( api_url, 'b2_copy_part', account_auth_token, sourceFileId=source_file_id, largeFileId=large_file_id, partNumber=part_number, **kwargs, ) except AccessDenied: raise SSECKeyError() def set_bucket_notification_rules( self, api_url: str, account_auth_token: str, bucket_id: str, rules: list[NotificationRule] ) -> list[NotificationRuleResponse]: return self._post_json( api_url, 'b2_set_bucket_notification_rules', account_auth_token, **{ 'bucketId': bucket_id, 'eventNotificationRules': rules, }, )['eventNotificationRules'] def get_bucket_notification_rules( self, api_url: str, account_auth_token: str, bucket_id: str ) -> list[NotificationRuleResponse]: return self._get_json( api_url, 'b2_get_bucket_notification_rules', account_auth_token, **{ 'bucketId': bucket_id, }, )['eventNotificationRules'] def _add_range_header(headers, range_): if range_ is not None: assert len(range_) == 2, range_ assert (range_[0] + 0) <= (range_[1] + 0), range_ # not strings assert range_[0] >= 0, range_ headers['Range'] = 'bytes=%d-%d' % range_ b2-sdk-python-2.8.0/b2sdk/_internal/raw_simulator.py000066400000000000000000002363521474454370000223670ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/raw_simulator.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import collections import dataclasses import io import logging import random import re import threading import time from contextlib import contextmanager, suppress from typing import Iterable from requests.structures import CaseInsensitiveDict from .b2http import ResponseContextManager from .encryption.setting import EncryptionMode, EncryptionSetting from .exception import ( AccessDenied, BadJson, BadRequest, BadUploadUrl, ChecksumMismatch, Conflict, CopySourceTooBig, DisablingFileLockNotSupported, DuplicateBucketName, FileNotPresent, FileSha1Mismatch, InvalidAuthToken, InvalidMetadataDirective, MissingPart, NonExistentBucket, PartSha1Mismatch, ResourceNotFound, SourceReplicationConflict, SSECKeyError, Unauthorized, UnsatisfiableRange, ) from .file_lock import ( NO_RETENTION_BUCKET_SETTING, BucketRetentionSetting, FileRetentionSetting, LegalHold, RetentionMode, ) from .file_version import UNVERIFIED_CHECKSUM_PREFIX from .http_constants import FILE_INFO_HEADER_PREFIX, HEX_DIGITS_AT_END from .raw_api import ( ALL_CAPABILITIES, AbstractRawApi, LifecycleRule, MetadataDirectiveMode, NotificationRule, NotificationRuleResponse, ) from .replication.setting import ReplicationConfiguration from .replication.types import ReplicationStatus from .stream.hashing import StreamWithHash from .utils import ConcurrentUsedAuthTokenGuard, b2_url_decode, b2_url_encode, hex_sha1_of_bytes logger = logging.getLogger(__name__) def get_bytes_range(data_bytes, bytes_range): """Slice bytes array using bytes range""" if bytes_range is None: return data_bytes if bytes_range[0] > bytes_range[1]: raise UnsatisfiableRange() if bytes_range[0] < 0: raise UnsatisfiableRange() if bytes_range[1] > len(data_bytes): raise UnsatisfiableRange() return data_bytes[bytes_range[0] : bytes_range[1] + 1] class KeySimulator: """ Hold information about one application key, which can be either a master application key, or one created with create_key(). """ def __init__( self, account_id, name, application_key_id, key, capabilities, expiration_timestamp_or_none, bucket_id_or_none, bucket_name_or_none, name_prefix_or_none, ): self.name = name self.account_id = account_id self.application_key_id = application_key_id self.key = key self.capabilities = capabilities self.expiration_timestamp_or_none = expiration_timestamp_or_none self.bucket_id_or_none = bucket_id_or_none self.bucket_name_or_none = bucket_name_or_none self.name_prefix_or_none = name_prefix_or_none def as_key(self): return dict( accountId=self.account_id, bucketId=self.bucket_id_or_none, applicationKeyId=self.application_key_id, capabilities=self.capabilities, expirationTimestamp=self.expiration_timestamp_or_none and self.expiration_timestamp_or_none * 1000, keyName=self.name, namePrefix=self.name_prefix_or_none, ) def as_created_key(self): """ Return the dict returned by b2_create_key. This is just like the one for b2_list_keys, but also includes the secret key. """ result = self.as_key() result['applicationKey'] = self.key return result def get_allowed(self): """ Return the 'allowed' structure to include in the response from b2_authorize_account. """ return dict( bucketId=self.bucket_id_or_none, bucketName=self.bucket_name_or_none, capabilities=self.capabilities, namePrefix=self.name_prefix_or_none, ) class PartSimulator: def __init__(self, file_id, part_number, content_length, content_sha1, part_data): self.file_id = file_id self.part_number = part_number self.content_length = content_length self.content_sha1 = content_sha1 self.part_data = part_data def as_list_parts_dict(self): return dict( fileId=self.file_id, partNumber=self.part_number, contentLength=self.content_length, contentSha1=self.content_sha1, ) class FileSimulator: """ One of three: an unfinished large file, a finished file, or a deletion marker. """ CHECK_ENCRYPTION = True SPECIAL_FILE_INFOS = { # when downloading, these file info keys are translated to specialized headers 'b2-content-disposition': 'Content-Disposition', 'b2-content-language': 'Content-Language', 'b2-expires': 'Expires', 'b2-cache-control': 'Cache-Control', 'b2-content-encoding': 'Content-Encoding', } def __init__( self, account_id, bucket, file_id, action, name, content_type, content_sha1, file_info, data_bytes, upload_timestamp, range_=None, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold = LegalHold.UNSET, replication_status: ReplicationStatus | None = None, ): if action == 'hide': assert server_side_encryption is None else: assert server_side_encryption is not None self.account_id = account_id self.bucket = bucket self.file_id = file_id self.action = action self.name = name if data_bytes is not None: self.content_length = len(data_bytes) self.content_type = content_type self.content_sha1 = content_sha1 if content_sha1 and content_sha1 != 'none' and len(content_sha1) != 40: raise ValueError(content_sha1) self.file_info = file_info self.data_bytes = data_bytes self.upload_timestamp = upload_timestamp self.range_ = range_ self.server_side_encryption = server_side_encryption self.file_retention = file_retention self.legal_hold = legal_hold if legal_hold is not None else LegalHold.UNSET self.replication_status = replication_status if action == 'start': self.parts = [] @classmethod @contextmanager def dont_check_encryption(cls): cls.CHECK_ENCRYPTION = False yield cls.CHECK_ENCRYPTION = True def sort_key(self): """ Return a key that can be used to sort the files in a bucket in the order that b2_list_file_versions returns them. """ return (self.name, self.file_id) def as_download_headers( self, account_auth_token_or_none: str | None = None, range_: tuple[int, int] | None = None ) -> dict[str, str]: if self.data_bytes is None: content_length = 0 elif range_ is not None: if range_[1] >= len(self.data_bytes): # requested too much content_length = len(self.data_bytes) else: content_length = range_[1] - range_[0] + 1 else: content_length = len(self.data_bytes) headers = CaseInsensitiveDict( { 'content-length': str(content_length), 'content-type': self.content_type, 'x-bz-content-sha1': self.content_sha1, 'x-bz-upload-timestamp': str(self.upload_timestamp), 'x-bz-file-id': self.file_id, 'x-bz-file-name': b2_url_encode(self.name), } ) for key, value in self.file_info.items(): key_lower = key.lower() if key_lower in self.SPECIAL_FILE_INFOS: headers[self.SPECIAL_FILE_INFOS[key_lower]] = value else: headers[FILE_INFO_HEADER_PREFIX + key] = b2_url_encode(value) if account_auth_token_or_none is not None and self.bucket.is_file_lock_enabled: not_permitted = [] if not self.is_allowed_to_read_file_retention(account_auth_token_or_none): not_permitted.append('X-Bz-File-Retention-Mode') not_permitted.append('X-Bz-File-Retain-Until-Timestamp') else: if self.file_retention is not None: self.file_retention.add_to_to_upload_headers(headers) if not self.is_allowed_to_read_file_legal_hold(account_auth_token_or_none): not_permitted.append('X-Bz-File-Legal-Hold') else: headers['X-Bz-File-Legal-Hold'] = self.legal_hold and 'on' or 'off' if not_permitted: headers['X-Bz-Client-Unauthorized-To-Read'] = ','.join(not_permitted) if self.server_side_encryption is not None: if self.server_side_encryption.mode == EncryptionMode.SSE_B2: headers['X-Bz-Server-Side-Encryption'] = self.server_side_encryption.algorithm.value elif self.server_side_encryption.mode == EncryptionMode.SSE_C: headers['X-Bz-Server-Side-Encryption-Customer-Algorithm'] = ( self.server_side_encryption.algorithm.value ) headers['X-Bz-Server-Side-Encryption-Customer-Key-Md5'] = ( self.server_side_encryption.key.key_md5() ) elif self.server_side_encryption.mode in (EncryptionMode.NONE, EncryptionMode.UNKNOWN): pass else: raise ValueError(f'Unsupported encryption mode: {self.server_side_encryption.mode}') if range_ is not None: headers['Content-Range'] = 'bytes %d-%d/%d' % ( range_[0], range_[0] + content_length - 1, len(self.data_bytes), ) return headers def as_upload_result(self, account_auth_token): result = dict( fileId=self.file_id, fileName=self.name, accountId=self.account_id, bucketId=self.bucket.bucket_id, contentLength=len(self.data_bytes) if self.data_bytes is not None else 0, contentType=self.content_type, contentSha1=self.content_sha1, fileInfo=self.file_info, action=self.action, uploadTimestamp=self.upload_timestamp, replicationStatus=self.replication_status and self.replication_status.value, ) if self.server_side_encryption is not None: result['serverSideEncryption'] = ( self.server_side_encryption.serialize_to_json_for_request() ) result['fileRetention'] = self._file_retention_dict(account_auth_token) result['legalHold'] = self._legal_hold_dict(account_auth_token) return result def as_list_files_dict(self, account_auth_token): result = dict( accountId=self.account_id, bucketId=self.bucket.bucket_id, fileId=self.file_id, fileName=self.name, contentLength=len(self.data_bytes) if self.data_bytes is not None else 0, contentType=self.content_type, contentSha1=self.content_sha1, fileInfo=self.file_info, action=self.action, uploadTimestamp=self.upload_timestamp, replicationStatus=self.replication_status and self.replication_status.value, ) if self.server_side_encryption is not None: result['serverSideEncryption'] = ( self.server_side_encryption.serialize_to_json_for_request() ) result['fileRetention'] = self._file_retention_dict(account_auth_token) result['legalHold'] = self._legal_hold_dict(account_auth_token) return result def is_allowed_to_read_file_retention(self, account_auth_token): return self.bucket._check_capability(account_auth_token, 'readFileRetentions') def is_allowed_to_read_file_legal_hold(self, account_auth_token): return self.bucket._check_capability(account_auth_token, 'readFileLegalHolds') def as_start_large_file_result(self, account_auth_token): result = dict( fileId=self.file_id, fileName=self.name, accountId=self.account_id, bucketId=self.bucket.bucket_id, contentType=self.content_type, fileInfo=self.file_info, uploadTimestamp=self.upload_timestamp, replicationStatus=self.replication_status and self.replication_status.value, ) if self.server_side_encryption is not None: result['serverSideEncryption'] = ( self.server_side_encryption.serialize_to_json_for_request() ) result['fileRetention'] = self._file_retention_dict(account_auth_token) result['legalHold'] = self._legal_hold_dict(account_auth_token) return result def _file_retention_dict(self, account_auth_token): if not self.is_allowed_to_read_file_retention(account_auth_token): return { 'isClientAuthorizedToRead': False, 'value': None, } file_lock_configuration = {'isClientAuthorizedToRead': True} if self.file_retention is None: file_lock_configuration['value'] = {'mode': None} else: file_lock_configuration['value'] = {'mode': self.file_retention.mode.value} if self.file_retention.retain_until is not None: file_lock_configuration['value']['retainUntilTimestamp'] = ( self.file_retention.retain_until ) return file_lock_configuration def _legal_hold_dict(self, account_auth_token): if not self.is_allowed_to_read_file_legal_hold(account_auth_token): return { 'isClientAuthorizedToRead': False, 'value': None, } return { 'isClientAuthorizedToRead': True, 'value': self.legal_hold.value, } def add_part(self, part_number, part): while len(self.parts) < part_number + 1: self.parts.append(None) self.parts[part_number] = part def finish(self, part_sha1_array): last_part_number = max(part.part_number for part in self.parts if part is not None) for part_number in range(1, last_part_number + 1): if self.parts[part_number] is None: raise MissingPart(part_number) my_part_sha1_array = [ self.parts[part_number].content_sha1 for part_number in range(1, last_part_number + 1) ] if part_sha1_array != my_part_sha1_array: raise ChecksumMismatch( 'sha1', expected=str(part_sha1_array), actual=str(my_part_sha1_array) ) self.data_bytes = b''.join( self.parts[part_number].part_data for part_number in range(1, last_part_number + 1) ) self.content_length = len(self.data_bytes) self.action = 'upload' def is_visible(self): """ Does this file show up in b2_list_file_names? """ return self.action == 'upload' def list_parts(self, start_part_number, max_part_count): start_part_number = start_part_number or 1 max_part_count = max_part_count or 100 parts = [ part.as_list_parts_dict() for part in self.parts if part is not None and start_part_number <= part.part_number ] if len(parts) <= max_part_count: next_part_number = None else: next_part_number = parts[max_part_count]['partNumber'] parts = parts[:max_part_count] return dict(parts=parts, nextPartNumber=next_part_number) def check_encryption(self, request_encryption: EncryptionSetting | None): if not self.CHECK_ENCRYPTION: return file_mode, file_secret = self._get_encryption_mode_and_secret(self.server_side_encryption) request_mode, request_secret = self._get_encryption_mode_and_secret(request_encryption) if file_mode in (None, EncryptionMode.NONE): assert request_mode in (None, EncryptionMode.NONE) elif file_mode == EncryptionMode.SSE_B2: assert request_mode in (None, EncryptionMode.NONE, EncryptionMode.SSE_B2) elif file_mode == EncryptionMode.SSE_C: if request_mode != EncryptionMode.SSE_C or file_secret != request_secret: raise SSECKeyError() else: raise ValueError('Unsupported EncryptionMode: %s' % (file_mode)) def _get_encryption_mode_and_secret(self, encryption: EncryptionSetting | None): if encryption is None: return None, None mode = encryption.mode if encryption.key is None: secret = None else: secret = encryption.key.secret return mode, secret @dataclasses.dataclass class FakeRequest: url: str headers: CaseInsensitiveDict @dataclasses.dataclass class FakeRaw: data_bytes: bytes _position: int = 0 def tell(self): return self._position def read(self, size): data = self.data_bytes[self._position : self._position + size] self._position += len(data) return data class FakeResponse: def __init__(self, account_auth_token_or_none, file_sim, url, range_=None): self.raw = FakeRaw(file_sim.data_bytes) self.headers = file_sim.as_download_headers(account_auth_token_or_none, range_) self.url = url self.range_ = range_ if range_ is not None: self.data_bytes = self.data_bytes[range_[0] : range_[1] + 1] @property def data_bytes(self): return self.raw.data_bytes @data_bytes.setter def data_bytes(self, value): self.raw = FakeRaw(value) def iter_content(self, chunk_size=1): rnd = random.Random(self.url) while True: chunk = self.raw.read(chunk_size) if chunk: time.sleep(rnd.random() * 0.01) yield chunk else: break @property def request(self): headers = CaseInsensitiveDict() if self.range_ is not None: headers['Range'] = '{}-{}'.format(*self.range_) return FakeRequest(self.url, headers) def close(self): pass def __enter__(self): return self def __exit__(self, *args): self.close() class BucketSimulator: # File IDs start at 9999 and count down, so they sort in the order # returned by list_file_versions. The IDs are strings. FIRST_FILE_NUMBER = 9999 FIRST_FILE_ID = str(FIRST_FILE_NUMBER) FILE_SIMULATOR_CLASS = FileSimulator RESPONSE_CLASS = FakeResponse MAX_SIMPLE_COPY_SIZE = 200 # should be same as RawSimulator.MIN_PART_SIZE def __init__( self, api, account_id, bucket_id, bucket_name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, options_set=None, default_server_side_encryption=None, is_file_lock_enabled: bool | None = None, replication: ReplicationConfiguration | None = None, ): assert bucket_type in ['allPrivate', 'allPublic'] self.api = api self.account_id = account_id self.bucket_name = bucket_name self.bucket_id = bucket_id self.bucket_type = bucket_type self.bucket_info = bucket_info or {} self.cors_rules = cors_rules or [] self.lifecycle_rules = lifecycle_rules or [] self._notification_rules = [] self.options_set = options_set or set() self.revision = 1 self.upload_url_counter = iter(range(200)) # File IDs count down, so that the most recent will come first when they are sorted. self.file_id_counter = iter(range(self.FIRST_FILE_NUMBER, 0, -1)) self.upload_timestamp_counter = iter(range(5000, 9999)) self.file_id_to_file: dict[str, FileSimulator] = dict() self.file_name_and_id_to_file: dict[tuple[str, str], FileSimulator] = dict() if default_server_side_encryption is None: default_server_side_encryption = EncryptionSetting(mode=EncryptionMode.NONE) self.default_server_side_encryption = default_server_side_encryption self.is_file_lock_enabled = is_file_lock_enabled self.default_retention = NO_RETENTION_BUCKET_SETTING self.replication = replication if self.replication is not None: assert ( self.replication.asReplicationSource is None or self.replication.asReplicationSource.rules ) assert ( self.replication.asReplicationDestination is None or self.replication.asReplicationDestination.sourceToDestinationKeyMapping ) def get_file(self, file_id, file_name) -> FileSimulator: try: return self.file_name_and_id_to_file[(file_name, file_id)] except KeyError: raise FileNotPresent(file_id_or_name=file_id) def is_allowed_to_read_bucket_encryption_setting(self, account_auth_token): return self._check_capability(account_auth_token, 'readBucketEncryption') def is_allowed_to_read_bucket_retention(self, account_auth_token): return self._check_capability(account_auth_token, 'readBucketRetentions') def _check_capability(self, account_auth_token, capability): try: key = self.api.auth_token_to_key[account_auth_token] except KeyError: # looks like it's an upload token # fortunately BucketSimulator makes it easy to retrieve the true account_auth_token # from an upload url real_auth_token = account_auth_token.split('/')[-1] key = self.api.auth_token_to_key[real_auth_token] capabilities = key.get_allowed()['capabilities'] return capability in capabilities def bucket_dict(self, account_auth_token): default_sse = {'isClientAuthorizedToRead': False} if self.is_allowed_to_read_bucket_encryption_setting(account_auth_token): default_sse['isClientAuthorizedToRead'] = True default_sse['value'] = {'mode': self.default_server_side_encryption.mode.value} if self.default_server_side_encryption.algorithm is not None: default_sse['value']['algorithm'] = ( self.default_server_side_encryption.algorithm.value ) else: default_sse['value'] = {'mode': EncryptionMode.UNKNOWN.value} if self.is_allowed_to_read_bucket_retention(account_auth_token): file_lock_configuration = { 'isClientAuthorizedToRead': True, 'value': { 'defaultRetention': { 'mode': self.default_retention.mode.value, 'period': self.default_retention.period.as_dict() if self.default_retention.period else None, }, 'isFileLockEnabled': self.is_file_lock_enabled, }, } else: file_lock_configuration = {'isClientAuthorizedToRead': False, 'value': None} replication = self.replication and { 'isClientAuthorizedToRead': True, 'value': self.replication.as_dict(), } return dict( accountId=self.account_id, bucketName=self.bucket_name, bucketId=self.bucket_id, bucketType=self.bucket_type, bucketInfo=self.bucket_info, corsRules=self.cors_rules, lifecycleRules=self.lifecycle_rules, options=self.options_set, revision=self.revision, defaultServerSideEncryption=default_sse, fileLockConfiguration=file_lock_configuration, replicationConfiguration=replication, ) def cancel_large_file(self, file_id): file_sim = self.file_id_to_file[file_id] key = (file_sim.name, file_id) del self.file_name_and_id_to_file[key] del self.file_id_to_file[file_id] return dict( accountId=self.account_id, bucketId=self.bucket_id, fileId=file_id, fileName=file_sim.name, ) def delete_file_version( self, account_auth_token, file_id, file_name, bypass_governance: bool = False ): key = (file_name, file_id) file_sim = self.get_file(file_id, file_name) if file_sim.file_retention: if file_sim.file_retention.retain_until and file_sim.file_retention.retain_until > int( time.time() ): if file_sim.file_retention.mode == RetentionMode.COMPLIANCE: raise AccessDenied() elif file_sim.file_retention.mode == RetentionMode.GOVERNANCE: if not bypass_governance: raise AccessDenied() if not self._check_capability(account_auth_token, 'bypassGovernance'): raise AccessDenied() del self.file_name_and_id_to_file[key] del self.file_id_to_file[file_id] return dict(fileId=file_id, fileName=file_name, uploadTimestamp=file_sim.upload_timestamp) def download_file_by_id( self, account_auth_token_or_none, file_id, url, range_=None, encryption: EncryptionSetting | None = None, ): file_sim = self.file_id_to_file[file_id] file_sim.check_encryption(encryption) return self._download_file_sim(account_auth_token_or_none, file_sim, url, range_=range_) def download_file_by_name( self, account_auth_token_or_none, file_name, url, range_=None, encryption: EncryptionSetting | None = None, ): files = self.list_file_names(self.api.current_token, file_name, 1)[ 'files' ] # token is not important here if len(files) == 0: raise FileNotPresent(file_id_or_name=file_name) file_dict = files[0] if file_dict['fileName'] != file_name: raise FileNotPresent(file_id_or_name=file_name) file_sim = self.file_name_and_id_to_file[(file_name, file_dict['fileId'])] if not file_sim.is_visible(): raise FileNotPresent(file_id_or_name=file_name) file_sim.check_encryption(encryption) return self._download_file_sim(account_auth_token_or_none, file_sim, url, range_=range_) def _download_file_sim(self, account_auth_token_or_none, file_sim, url, range_=None): return ResponseContextManager( self.RESPONSE_CLASS( account_auth_token_or_none, file_sim, url, range_, ) ) def finish_large_file(self, account_auth_token, file_id, part_sha1_array): file_sim = self.file_id_to_file[file_id] file_sim.finish(part_sha1_array) return file_sim.as_upload_result(account_auth_token) def get_file_info_by_id(self, account_auth_token, file_id): return self.file_id_to_file[file_id].as_upload_result(account_auth_token) def get_file_info_by_name(self, account_auth_token, file_name): # Sorting files by name and ID, so lower ID (newer upload) is returned first. for (name, id), file in sorted(self.file_name_and_id_to_file.items()): if file_name == name: return file.as_download_headers(account_auth_token_or_none=account_auth_token) raise FileNotPresent(file_id_or_name=file_name, bucket_name=self.bucket_name) def get_upload_url(self, account_auth_token): upload_id = next(self.upload_url_counter) upload_url = 'https://upload.example.com/%s/%d/%s' % ( self.bucket_id, upload_id, account_auth_token, ) return dict(bucketId=self.bucket_id, uploadUrl=upload_url, authorizationToken=upload_url) def get_upload_part_url(self, account_auth_token, file_id): upload_url = 'https://upload.example.com/part/%s/%d/%s' % ( file_id, random.randint(1, 10**9), account_auth_token, ) return dict(bucketId=self.bucket_id, uploadUrl=upload_url, authorizationToken=upload_url) def hide_file(self, account_auth_token, file_name): file_id = self._next_file_id() file_sim = self.FILE_SIMULATOR_CLASS( self.account_id, self, file_id, 'hide', file_name, None, 'none', {}, b'', next(self.upload_timestamp_counter), ) self.file_id_to_file[file_id] = file_sim self.file_name_and_id_to_file[file_sim.sort_key()] = file_sim return file_sim.as_list_files_dict(account_auth_token) def update_file_retention( self, account_auth_token, file_id, file_name, file_retention: FileRetentionSetting, bypass_governance: bool = False, ): file_sim = self.file_id_to_file[file_id] assert self.is_file_lock_enabled assert file_sim.name == file_name # TODO: check bypass etc file_sim.file_retention = file_retention return { 'fileId': file_id, 'fileName': file_name, 'fileRetention': file_sim.file_retention.serialize_to_json_for_request(), } def update_file_legal_hold( self, account_auth_token, file_id, file_name, legal_hold: LegalHold, ): file_sim = self.file_id_to_file[file_id] assert self.is_file_lock_enabled assert file_sim.name == file_name file_sim.legal_hold = legal_hold return { 'fileId': file_id, 'fileName': file_name, 'legalHold': legal_hold.to_server(), } def copy_file( self, account_auth_token, file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_bucket_id=None, destination_server_side_encryption: EncryptionSetting | None = None, source_server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, ): if metadata_directive is not None: assert metadata_directive in tuple(MetadataDirectiveMode), metadata_directive if metadata_directive is MetadataDirectiveMode.COPY and ( content_type is not None or file_info is not None ): raise InvalidMetadataDirective( 'content_type and file_info should be None when metadata_directive is COPY' ) elif metadata_directive is MetadataDirectiveMode.REPLACE and content_type is None: raise InvalidMetadataDirective( 'content_type cannot be None when metadata_directive is REPLACE' ) file_sim = self.file_id_to_file[file_id] file_sim.check_encryption(source_server_side_encryption) new_file_id = self._next_file_id() data_bytes = get_bytes_range(file_sim.data_bytes, bytes_range) if len(data_bytes) > self.MAX_SIMPLE_COPY_SIZE: raise CopySourceTooBig( 'Copy source too big: %i' % (len(data_bytes),), 'bad_request', len(data_bytes), ) destination_bucket = self.api.bucket_id_to_bucket.get(destination_bucket_id, self) sse = destination_server_side_encryption or self.default_server_side_encryption copy_file_sim = self.FILE_SIMULATOR_CLASS( self.account_id, destination_bucket, new_file_id, 'upload', new_file_name, file_sim.content_type, hex_sha1_of_bytes( data_bytes ), # we hash here again because bytes_range may not cover the full source file_sim.file_info, data_bytes, next(self.upload_timestamp_counter), server_side_encryption=sse, file_retention=file_retention, legal_hold=legal_hold, ) destination_bucket.file_id_to_file[copy_file_sim.file_id] = copy_file_sim destination_bucket.file_name_and_id_to_file[copy_file_sim.sort_key()] = copy_file_sim if metadata_directive is MetadataDirectiveMode.REPLACE: copy_file_sim.content_type = content_type copy_file_sim.file_info = file_info or file_sim.file_info ## long term storage of that file has action="upload", but here we need to return action="copy", just this once # class TestFileVersionFactory(FileVersionFactory): # FILE_VERSION_CLASS = self.FILE_SIMULATOR_CLASS # file_version_dict = copy_file_sim.as_upload_result(account_auth_token) # del file_version_dict['action'] # print(file_version_dict) # copy_file_sim_with_action_copy = TestFileVersionFactory(self.api).from_api_response(file_version_dict, force_action='copy') # return copy_file_sim_with_action_copy # TODO: the code above cannot be used right now because FileSimulator.__init__ is incompatible with FileVersionFactory / FileVersion.__init__ - refactor is needed # for now we'll just return the newly constructed object with a copy action... return self.FILE_SIMULATOR_CLASS( self.account_id, destination_bucket, new_file_id, 'copy', new_file_name, copy_file_sim.content_type, copy_file_sim.content_sha1, copy_file_sim.file_info, data_bytes, copy_file_sim.upload_timestamp, server_side_encryption=sse, file_retention=file_retention, legal_hold=legal_hold, ) def list_file_names( self, account_auth_token, start_file_name=None, max_file_count=None, prefix=None, ): assert ( prefix is None or start_file_name is None or start_file_name.startswith(prefix) ), locals() start_file_name = start_file_name or '' max_file_count = max_file_count or 100 result_files = [] next_file_name = None prev_file_name = None for key in sorted(self.file_name_and_id_to_file): (file_name, file_id) = key assert file_id if start_file_name <= file_name and file_name != prev_file_name: if prefix is not None and not file_name.startswith(prefix): break prev_file_name = file_name file_sim = self.file_name_and_id_to_file[key] if file_sim.is_visible(): result_files.append(file_sim.as_list_files_dict(account_auth_token)) if len(result_files) == max_file_count: next_file_name = file_sim.name + ' ' break else: logger.debug('skipping invisible file during listing: %s', key) return dict(files=result_files, nextFileName=next_file_name) def list_file_versions( self, account_auth_token, start_file_name=None, start_file_id=None, max_file_count=None, prefix=None, ): assert ( prefix is None or start_file_name is None or start_file_name.startswith(prefix) ), locals() start_file_name = start_file_name or '' start_file_id = start_file_id or '' max_file_count = max_file_count or 100 result_files = [] next_file_name = None next_file_id = None for key in sorted(self.file_name_and_id_to_file): (file_name, file_id) = key if (start_file_name < file_name) or ( start_file_name == file_name and (start_file_id == '' or int(start_file_id) <= int(file_id)) ): file_sim = self.file_name_and_id_to_file[key] if prefix is not None and not file_name.startswith(prefix): break result_files.append(file_sim.as_list_files_dict(account_auth_token)) if len(result_files) == max_file_count: next_file_name = file_sim.name next_file_id = str(int(file_id) + 1) break return dict(files=result_files, nextFileName=next_file_name, nextFileId=next_file_id) def list_parts(self, file_id, start_part_number, max_part_count): file_sim = self.file_id_to_file[file_id] return file_sim.list_parts(start_part_number, max_part_count) def list_unfinished_large_files( self, account_auth_token, start_file_id=None, max_file_count=None, prefix=None ): start_file_id = start_file_id or self.FIRST_FILE_ID max_file_count = max_file_count or 100 all_unfinished_ids = set( k for (k, v) in self.file_id_to_file.items() if v.action == 'start' and k <= start_file_id and (prefix is None or v.name.startswith(prefix)) ) ids_in_order = sorted(all_unfinished_ids, reverse=True) file_dict_list = [ file_sim.as_start_large_file_result(account_auth_token) for file_sim in ( self.file_id_to_file[file_id] for file_id in ids_in_order[:max_file_count] ) ] next_file_id = None if len(file_dict_list) == max_file_count: next_file_id = str(int(file_dict_list[-1]['fileId']) - 1) return dict(files=file_dict_list, nextFileId=next_file_id) def start_large_file( self, account_auth_token, file_name, content_type, file_info, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): file_id = self._next_file_id() sse = server_side_encryption or self.default_server_side_encryption if ( sse ): # FIXME: remove this part when RawApi<->Encryption adapters are implemented properly file_info = sse.add_key_id_to_file_info(file_info) upload_timestamp = next(self.upload_timestamp_counter) if custom_upload_timestamp is not None: upload_timestamp = custom_upload_timestamp file_sim = self.FILE_SIMULATOR_CLASS( self.account_id, self, file_id, 'start', file_name, content_type, 'none', file_info, None, upload_timestamp, server_side_encryption=sse, file_retention=file_retention, legal_hold=legal_hold, ) self.file_id_to_file[file_id] = file_sim self.file_name_and_id_to_file[file_sim.sort_key()] = file_sim return file_sim.as_start_large_file_result(account_auth_token) def _update_bucket( self, bucket_type=None, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, if_revision_is: int | None = None, default_server_side_encryption: EncryptionSetting | None = None, default_retention: BucketRetentionSetting | None = None, replication: ReplicationConfiguration | None = None, is_file_lock_enabled: bool | None = None, ): if if_revision_is is not None and self.revision != if_revision_is: raise Conflict() if is_file_lock_enabled is not None: if self.is_file_lock_enabled and not is_file_lock_enabled: raise DisablingFileLockNotSupported() if ( not self.is_file_lock_enabled and is_file_lock_enabled and self.replication and self.replication.is_source ): raise SourceReplicationConflict() self.is_file_lock_enabled = is_file_lock_enabled if bucket_type is not None: self.bucket_type = bucket_type if bucket_info is not None: self.bucket_info = bucket_info if cors_rules is not None: self.cors_rules = cors_rules if lifecycle_rules is not None: self.lifecycle_rules = lifecycle_rules if default_server_side_encryption is not None: self.default_server_side_encryption = default_server_side_encryption if default_retention: self.default_retention = default_retention if replication is not None: self.replication = replication self.revision += 1 return self.bucket_dict(self.api.current_token) def upload_file( self, upload_id: str, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_info: dict, data_stream, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): data_bytes = self._simulate_chunked_post(data_stream, content_length) assert len(data_bytes) == content_length if content_sha1 == HEX_DIGITS_AT_END: content_sha1 = data_bytes[-40:].decode() data_bytes = data_bytes[0:-40] content_length -= 40 elif len(content_sha1) != 40: raise ValueError(content_sha1) computed_sha1 = hex_sha1_of_bytes(data_bytes) if content_sha1 != computed_sha1: raise FileSha1Mismatch(file_name) if content_sha1 == 'do_not_verify': content_sha1 = UNVERIFIED_CHECKSUM_PREFIX + computed_sha1 file_id = self._next_file_id() encryption = server_side_encryption or self.default_server_side_encryption if ( encryption ): # FIXME: remove this part when RawApi<->Encryption adapters are implemented properly file_info = encryption.add_key_id_to_file_info(file_info) upload_timestamp = next(self.upload_timestamp_counter) if custom_upload_timestamp is not None: upload_timestamp = custom_upload_timestamp file_sim = self.FILE_SIMULATOR_CLASS( self.account_id, self, file_id, 'upload', file_name, content_type, content_sha1, file_info, data_bytes, upload_timestamp, server_side_encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, ) self.file_id_to_file[file_id] = file_sim self.file_name_and_id_to_file[file_sim.sort_key()] = file_sim return file_sim.as_upload_result(upload_auth_token) def upload_part( self, file_id, part_number, content_length, sha1_sum, input_stream, server_side_encryption: EncryptionSetting | None = None, ): part_data = self._simulate_chunked_post(input_stream, content_length) assert len(part_data) == content_length if sha1_sum == HEX_DIGITS_AT_END: sha1_sum = part_data[-40:].decode() part_data = part_data[0:-40] content_length -= 40 computed_sha1 = hex_sha1_of_bytes(part_data) if sha1_sum != computed_sha1: raise PartSha1Mismatch(file_id) file_sim = self.file_id_to_file[file_id] part = PartSimulator(file_sim.file_id, part_number, content_length, sha1_sum, part_data) file_sim.add_part(part_number, part) result = dict( fileId=file_id, partNumber=part_number, contentLength=content_length, contentSha1=sha1_sum, ) if server_side_encryption is not None: result['serverSideEncryption'] = server_side_encryption.serialize_to_json_for_request() return result def _simulate_chunked_post( self, stream, content_length, min_chunks=4, max_chunk_size=8096, simulate_retry=True ): chunk_size = max_chunk_size chunks_num = self._chunks_number(content_length, chunk_size) if chunks_num < min_chunks: chunk_size = max(content_length // min_chunks, 1) loop_count = 2 if simulate_retry else 1 stream_data = None for _ in range(loop_count): chunks = [] stream.seek(0) # we always do this in `do_post` in `b2http` so we want it here *always* while True: data = stream.read(chunk_size) chunks.append(data) if not data: break _stream_data = b''.join(chunks) if stream_data is not None: assert _stream_data == stream_data stream_data = _stream_data return stream_data def _chunks_number(self, content_length, chunk_size): chunks_number = content_length // chunk_size if content_length % chunk_size > 0: chunks_number = chunks_number + 1 return chunks_number def _next_file_id(self): return str(next(self.file_id_counter)) def get_notification_rules(self) -> list[NotificationRule]: return self._notification_rules def set_notification_rules( self, rules: Iterable[NotificationRule] ) -> list[NotificationRuleResponse]: old_rules_by_name = {rule['name']: rule for rule in self._notification_rules} new_rules: list[NotificationRuleResponse] = [] for rule in rules: for field in ('isSuspended', 'suspensionReason'): rule.pop(field, None) old_rule = old_rules_by_name.get(rule['name'], {'targetConfiguration': {}}) new_rule = { **{ 'isSuspended': False, 'suspensionReason': '', }, **old_rule, **rule, 'targetConfiguration': { **old_rule.get('targetConfiguration', {}), **rule.get('targetConfiguration', {}), }, } new_rules.append(new_rule) self._notification_rules = new_rules return self._notification_rules def simulate_notification_rule_suspension( self, rule_name: str, reason: str, is_suspended: bool | None = None ) -> None: for rule in self._notification_rules: if rule['name'] == rule_name: rule['isSuspended'] = bool(reason) if is_suspended is None else is_suspended rule['suspensionReason'] = reason return raise ResourceNotFound(f'Rule {rule_name} not found') class RawSimulator(AbstractRawApi): """ Implement the same interface as B2RawHTTPApi by simulating all of the calls and keeping state in memory. The intended use for this class is for unit tests that test things built on top of B2RawHTTPApi. """ BUCKET_SIMULATOR_CLASS = BucketSimulator API_URL = 'http://api.example.com' S3_API_URL = 'http://s3.api.example.com' DOWNLOAD_URL = 'http://download.example.com' MIN_PART_SIZE = 200 MAX_PART_ID = 10000 # This is the maximum duration in seconds that an application key can be valid (1000 days). MAX_DURATION_IN_SECONDS = 86400000 UPLOAD_PART_MATCHER = re.compile('https://upload.example.com/part/([^/]*)') UPLOAD_URL_MATCHER = re.compile(r'https://upload.example.com/([^/]*)/([^/]*)') DOWNLOAD_URL_MATCHER = re.compile( DOWNLOAD_URL + '(?:' + '|'.join( ( r'/b2api/v[0-9]+/b2_download_file_by_id\?fileId=(?P[^/]+)', '/file/(?P[^/]+)/(?P.+)', ) ) + ')$' ) def __init__(self, b2_http=None): # Map from application_key_id to KeySimulator. # The entry for the master application key ID is for the master application # key for the account, and the entries with non-master application keys # are for keys created b2 createKey(). self.key_id_to_key = dict() # Map from auth token to the KeySimulator for it. self.auth_token_to_key = dict() # Set of auth tokens that have expired self.expired_auth_tokens = set() # Map from auth token to a lock that upload procedure acquires # when utilizing the token self.currently_used_auth_tokens = collections.defaultdict(threading.Lock) # Counter for generating auth tokens. self.auth_token_counter = 0 # Counter for generating account IDs an their matching master application keys. self.account_counter = 0 self.bucket_name_to_bucket: dict[str, BucketSimulator] = dict() self.bucket_id_to_bucket: dict[str, BucketSimulator] = dict() self.bucket_id_counter = iter(range(100)) self.file_id_to_bucket_id: dict[str, str] = {} self.all_application_keys = [] self.app_key_counter = 0 self.upload_errors = [] def expire_auth_token(self, auth_token): """ Simulate the auth token expiring. The next call that tries to use this auth token will get an auth_token_expired error. """ assert auth_token in self.auth_token_to_key self.expired_auth_tokens.add(auth_token) def create_account(self): """ Simulate creating an account. Return (accountId, masterApplicationKey) for a newly created account. """ # Pick the IDs for the account and the key account_id = 'account-%d' % (self.account_counter,) master_key = 'masterKey-%d' % (self.account_counter,) self.account_counter += 1 # Create the key self.key_id_to_key[account_id] = KeySimulator( account_id=account_id, name='master', application_key_id=account_id, key=master_key, capabilities=ALL_CAPABILITIES, expiration_timestamp_or_none=None, bucket_id_or_none=None, bucket_name_or_none=None, name_prefix_or_none=None, ) # Return the info return (account_id, master_key) def set_upload_errors(self, errors): """ Store a sequence of exceptions to raise on upload. Each one will be raised in turn, until they are all gone. Then the next upload will succeed. """ assert len(self.upload_errors) == 0 self.upload_errors = errors def authorize_account(self, realm_url, application_key_id, application_key): key_sim = self.key_id_to_key.get(application_key_id) if key_sim is None: raise InvalidAuthToken('application key ID not valid', 'unauthorized') if application_key != key_sim.key: raise InvalidAuthToken('secret key is wrong', 'unauthorized') auth_token = 'auth_token_%d' % (self.auth_token_counter,) self.current_token = auth_token self.auth_token_counter += 1 self.auth_token_to_key[auth_token] = key_sim allowed = key_sim.get_allowed() bucketId = allowed.get('bucketId') if (bucketId is not None) and (bucketId in self.bucket_id_to_bucket): allowed['bucketName'] = self.bucket_id_to_bucket[bucketId].bucket_name else: allowed['bucketName'] = None return dict( accountId=key_sim.account_id, authorizationToken=auth_token, apiInfo=dict( groupsApi=dict(), storageApi=dict( apiUrl=self.API_URL, downloadUrl=self.DOWNLOAD_URL, recommendedPartSize=self.MIN_PART_SIZE, absoluteMinimumPartSize=self.MIN_PART_SIZE, allowed=allowed, s3ApiUrl=self.S3_API_URL, bucketId=allowed['bucketId'], bucketName=allowed['bucketName'], capabilities=allowed['capabilities'], namePrefix=allowed['namePrefix'], ), ), ) def cancel_large_file(self, api_url, account_auth_token, file_id): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') return bucket.cancel_large_file(file_id) def create_bucket( self, api_url, account_auth_token, account_id, bucket_name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, default_server_side_encryption: EncryptionSetting | None = None, is_file_lock_enabled: bool | None = None, replication: ReplicationConfiguration | None = None, ): if not re.match(r'^[-a-zA-Z0-9]*$', bucket_name): raise BadJson('illegal bucket name: ' + bucket_name) self._assert_account_auth(api_url, account_auth_token, account_id, 'writeBuckets') if bucket_name in self.bucket_name_to_bucket: raise DuplicateBucketName(bucket_name) bucket_id = 'bucket_' + str(next(self.bucket_id_counter)) bucket = self.BUCKET_SIMULATOR_CLASS( self, account_id, bucket_id, bucket_name, bucket_type, bucket_info, cors_rules, lifecycle_rules, # watch out for options! default_server_side_encryption=default_server_side_encryption, is_file_lock_enabled=is_file_lock_enabled, replication=replication, ) self.bucket_name_to_bucket[bucket_name] = bucket self.bucket_id_to_bucket[bucket_id] = bucket return bucket.bucket_dict(account_auth_token) # TODO it should be an object, right? def create_key( self, api_url, account_auth_token, account_id, capabilities, key_name, valid_duration_seconds, bucket_id, name_prefix, ): if not re.match(r'^[A-Za-z0-9-]{1,100}$', key_name): raise BadJson('illegal key name: ' + key_name) if valid_duration_seconds is not None: if valid_duration_seconds < 1 or valid_duration_seconds > self.MAX_DURATION_IN_SECONDS: raise BadJson( 'valid duration must be greater than 0, and less than 1000 days in seconds' ) self._assert_account_auth(api_url, account_auth_token, account_id, 'writeKeys') if valid_duration_seconds is None: expiration_timestamp_or_none = None else: expiration_timestamp_or_none = int(time.time() + valid_duration_seconds) index = self.app_key_counter self.app_key_counter += 1 application_key_id = 'appKeyId%d' % (index,) app_key = 'appKey%d' % (index,) bucket_name_or_none = None if bucket_id is not None: # It is possible for bucketId to be filled and bucketName to be empty. # It can happen when the bucket was deleted. with suppress(NonExistentBucket): bucket_name_or_none = self._get_bucket_by_id(bucket_id).bucket_name key_sim = KeySimulator( account_id=account_id, name=key_name, application_key_id=application_key_id, key=app_key, capabilities=capabilities, expiration_timestamp_or_none=expiration_timestamp_or_none, bucket_id_or_none=bucket_id, bucket_name_or_none=bucket_name_or_none, name_prefix_or_none=name_prefix, ) self.key_id_to_key[application_key_id] = key_sim self.all_application_keys.append(key_sim) return key_sim.as_created_key() def delete_file_version( self, api_url, account_auth_token, file_id, file_name, bypass_governance: bool = False ): bucket_id = self.file_id_to_bucket_id.get(file_id) if not bucket_id: raise FileNotPresent(file_id_or_name=file_id) bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'deleteFiles') return bucket.delete_file_version(account_auth_token, file_id, file_name, bypass_governance) def update_file_retention( self, api_url, account_auth_token, file_id, file_name, file_retention: FileRetentionSetting, bypass_governance: bool = False, ): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) return bucket.update_file_retention( account_auth_token, file_id, file_name, file_retention, bypass_governance, ) def update_file_legal_hold( self, api_url, account_auth_token, file_id, file_name, legal_hold: bool, ): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) return bucket.update_file_legal_hold( account_auth_token, file_id, file_name, legal_hold, ) def delete_bucket(self, api_url, account_auth_token, account_id, bucket_id): self._assert_account_auth(api_url, account_auth_token, account_id, 'deleteBuckets') bucket = self._get_bucket_by_id(bucket_id) del self.bucket_name_to_bucket[bucket.bucket_name] del self.bucket_id_to_bucket[bucket_id] return bucket.bucket_dict(account_auth_token) def download_file_from_url( self, account_auth_token_or_none: str | None, url: str, range_: tuple[int, int] | None = None, encryption: EncryptionSetting | None = None, ): # TODO: check auth token if bucket is not public matcher = self.DOWNLOAD_URL_MATCHER.match(url) assert matcher is not None, url groupdict = matcher.groupdict() file_id = groupdict['file_id'] bucket_name = groupdict['bucket_name'] file_name = groupdict['file_name'] if file_id is not None: bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) return bucket.download_file_by_id( account_auth_token_or_none, file_id, range_=range_, url=url, encryption=encryption, ) elif bucket_name is not None and file_name is not None: bucket = self._get_bucket_by_name(bucket_name) return bucket.download_file_by_name( account_auth_token_or_none, b2_url_decode(file_name), range_=range_, url=url, encryption=encryption, ) else: assert False def delete_key(self, api_url, account_auth_token, application_key_id): assert api_url == self.API_URL key_sim = self.key_id_to_key.pop(application_key_id, None) if key_sim is None: raise BadRequest( f'application key does not exist: {application_key_id}', 'bad_request', ) self.all_application_keys = [ key for key in self.all_application_keys if key.application_key_id != application_key_id ] return key_sim.as_key() def finish_large_file(self, api_url, account_auth_token, file_id, part_sha1_array): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') return bucket.finish_large_file(account_auth_token, file_id, part_sha1_array) def get_download_authorization( self, api_url, account_auth_token, bucket_id, file_name_prefix, valid_duration_in_seconds ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'shareFiles') return { 'bucketId': bucket_id, 'fileNamePrefix': file_name_prefix, 'authorizationToken': 'fake_download_auth_token_%s_%s_%d' % ( bucket_id, b2_url_encode(file_name_prefix), valid_duration_in_seconds, ), } def get_file_info_by_id(self, api_url, account_auth_token, file_id): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) return bucket.get_file_info_by_id(account_auth_token, file_id) def get_file_info_by_name(self, api_url, account_auth_token, bucket_name, file_name): bucket = self._get_bucket_by_name(bucket_name) info = bucket.get_file_info_by_name(account_auth_token, file_name) return info def get_upload_url(self, api_url, account_auth_token, bucket_id): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') return self._get_bucket_by_id(bucket_id).get_upload_url(account_auth_token) def get_upload_part_url(self, api_url, account_auth_token, file_id): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') return self._get_bucket_by_id(bucket_id).get_upload_part_url(account_auth_token, file_id) def hide_file(self, api_url, account_auth_token, bucket_id, file_name): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') response = bucket.hide_file(account_auth_token, file_name) self.file_id_to_bucket_id[response['fileId']] = bucket_id return response def copy_file( self, api_url, account_auth_token, source_file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_bucket_id=None, destination_server_side_encryption=None, source_server_side_encryption=None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, ): bucket_id = self.file_id_to_bucket_id[source_file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') if destination_bucket_id: # TODO: Handle and raise proper exception after server docs get updated dest_bucket = self.bucket_id_to_bucket[destination_bucket_id] assert dest_bucket.account_id == bucket.account_id else: dest_bucket = bucket copy_file_sim = bucket.copy_file( account_auth_token, source_file_id, new_file_name, bytes_range, metadata_directive, content_type, file_info, destination_bucket_id, destination_server_side_encryption=destination_server_side_encryption, source_server_side_encryption=source_server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, ) return copy_file_sim.as_upload_result(account_auth_token) def copy_part( self, api_url, account_auth_token, source_file_id, large_file_id, part_number, bytes_range=None, destination_server_side_encryption: EncryptionSetting | None = None, source_server_side_encryption: EncryptionSetting | None = None, ): if ( destination_server_side_encryption is not None and destination_server_side_encryption.mode == EncryptionMode.SSE_B2 ): raise ValueError( 'unsupported sse mode for copy_part!' ) # SSE-B2 is only to be marked in b2_start_large_file src_bucket_id = self.file_id_to_bucket_id[source_file_id] src_bucket = self._get_bucket_by_id(src_bucket_id) dest_bucket_id = self.file_id_to_bucket_id[large_file_id] dest_bucket = self._get_bucket_by_id(dest_bucket_id) self._assert_account_auth(api_url, account_auth_token, dest_bucket.account_id, 'writeFiles') file_sim = src_bucket.file_id_to_file[source_file_id] file_sim.check_encryption(source_server_side_encryption) data_bytes = get_bytes_range(file_sim.data_bytes, bytes_range) data_stream = StreamWithHash(io.BytesIO(data_bytes), len(data_bytes)) content_length = len(data_stream) sha1_sum = HEX_DIGITS_AT_END return dest_bucket.upload_part( large_file_id, part_number, content_length, sha1_sum, data_stream, server_side_encryption=destination_server_side_encryption, ) def list_buckets( self, api_url, account_auth_token, account_id, bucket_id=None, bucket_name=None ): # First, map the bucket name to a bucket_id, so that we can check auth. if bucket_name is None: bucket_id_for_auth = bucket_id else: bucket_id_for_auth = self._get_bucket_id_or_none_for_bucket_name(bucket_name) self._assert_account_auth( api_url, account_auth_token, account_id, 'listBuckets', bucket_id_for_auth ) # Do the query sorted_buckets = [ self.bucket_name_to_bucket[name] for name in sorted(self.bucket_name_to_bucket) ] bucket_list = [ bucket.bucket_dict(account_auth_token) for bucket in sorted_buckets if self._bucket_matches(bucket, bucket_id, bucket_name) ] return dict(buckets=bucket_list) def _get_bucket_id_or_none_for_bucket_name(self, bucket_name): for bucket in self.bucket_name_to_bucket.values(): if bucket.bucket_name == bucket_name: return bucket.bucket_id def _bucket_matches(self, bucket, bucket_id, bucket_name): return (bucket_id is None or bucket.bucket_id == bucket_id) and ( bucket_name is None or bucket.bucket_name == bucket_name ) def list_file_names( self, api_url, account_auth_token, bucket_id, start_file_name=None, max_file_count=None, prefix=None, ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth( api_url, account_auth_token, bucket.account_id, 'listFiles', bucket_id=bucket_id, file_name=prefix, ) return bucket.list_file_names(account_auth_token, start_file_name, max_file_count, prefix) def list_file_versions( self, api_url, account_auth_token, bucket_id, start_file_name=None, start_file_id=None, max_file_count=None, prefix=None, ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth( api_url, account_auth_token, bucket.account_id, 'listFiles', bucket_id=bucket_id, file_name=prefix, ) return bucket.list_file_versions( account_auth_token, start_file_name, start_file_id, max_file_count, prefix, ) def list_keys( self, api_url, account_auth_token, account_id, max_key_count=1000, start_application_key_id=None, ): self._assert_account_auth(api_url, account_auth_token, account_id, 'listKeys') next_application_key_id = None all_keys_sorted = sorted(self.all_application_keys, key=lambda key: key.application_key_id) if start_application_key_id is None: keys = all_keys_sorted[:max_key_count] if max_key_count < len(all_keys_sorted): next_application_key_id = all_keys_sorted[max_key_count].application_key_id else: keys = [] got_already = 0 for ind, key in enumerate(all_keys_sorted): if key.application_key_id >= start_application_key_id: keys.append(key) got_already += 1 if got_already == max_key_count: if ind < len(all_keys_sorted) - 1: next_application_key_id = all_keys_sorted[ind + 1].application_key_id break key_dicts = map(lambda key: key.as_key(), keys) return dict(keys=list(key_dicts), nextApplicationKeyId=next_application_key_id) def list_parts(self, api_url, account_auth_token, file_id, start_part_number, max_part_count): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') return bucket.list_parts(file_id, start_part_number, max_part_count) def list_unfinished_large_files( self, api_url, account_auth_token, bucket_id, start_file_id=None, max_file_count=None, prefix=None, ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth( api_url, account_auth_token, bucket.account_id, 'listFiles', file_name=prefix ) start_file_id = start_file_id or '' max_file_count = max_file_count or 100 return bucket.list_unfinished_large_files( account_auth_token, start_file_id, max_file_count, prefix ) def start_large_file( self, api_url, account_auth_token, bucket_id, file_name, content_type, file_info, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') result = bucket.start_large_file( account_auth_token, file_name, content_type, file_info, server_side_encryption, file_retention, legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) self.file_id_to_bucket_id[result['fileId']] = bucket_id return result def update_bucket( self, api_url, account_auth_token, account_id, bucket_id, bucket_type=None, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, if_revision_is=None, default_server_side_encryption: EncryptionSetting | None = None, default_retention: BucketRetentionSetting | None = None, replication: ReplicationConfiguration | None = None, is_file_lock_enabled: bool | None = None, ): assert ( bucket_type or bucket_info or cors_rules or lifecycle_rules or default_server_side_encryption or replication or is_file_lock_enabled is not None ) bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeBuckets') return bucket._update_bucket( bucket_type=bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, replication=replication, is_file_lock_enabled=is_file_lock_enabled, ) @classmethod def get_upload_file_headers( cls, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_info: dict, server_side_encryption: EncryptionSetting | None, file_retention: FileRetentionSetting | None, legal_hold: LegalHold | None, custom_upload_timestamp: int | None = None, ) -> dict: # fix to allow calculating headers on unknown key - only for simulation if ( server_side_encryption is not None and server_side_encryption.mode == EncryptionMode.SSE_C and server_side_encryption.key.secret is None ): server_side_encryption.key.secret = b'secret' return super().get_upload_file_headers( upload_auth_token=upload_auth_token, file_name=file_name, content_length=content_length, content_type=content_type, content_sha1=content_sha1, file_info=file_info, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) def upload_file( self, upload_url: str, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_info: dict, data_stream, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): with ConcurrentUsedAuthTokenGuard( self.currently_used_auth_tokens[upload_auth_token], upload_auth_token ): assert upload_url == upload_auth_token url_match = self.UPLOAD_URL_MATCHER.match(upload_url) if url_match is None: raise BadUploadUrl(upload_url) if self.upload_errors: raise self.upload_errors.pop(0) bucket_id, upload_id = url_match.groups() bucket = self._get_bucket_by_id(bucket_id) if server_side_encryption is not None: assert server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C, ) file_info = server_side_encryption.add_key_id_to_file_info(file_info) # we don't really need headers further on # but we still simulate their calculation _ = self.get_upload_file_headers( upload_auth_token=upload_auth_token, file_name=file_name, content_length=content_length, content_type=content_type, content_sha1=content_sha1, file_info=file_info, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) response = bucket.upload_file( upload_id, upload_auth_token, file_name, content_length, content_type, content_sha1, file_info, data_stream, server_side_encryption, file_retention, legal_hold, custom_upload_timestamp, ) file_id = response['fileId'] self.file_id_to_bucket_id[file_id] = bucket_id return response def upload_part( self, upload_url, upload_auth_token, part_number, content_length, sha1_sum, input_stream, server_side_encryption: EncryptionSetting | None = None, ): with ConcurrentUsedAuthTokenGuard( self.currently_used_auth_tokens[upload_auth_token], upload_auth_token ): url_match = self.UPLOAD_PART_MATCHER.match(upload_url) if url_match is None: raise BadUploadUrl(upload_url) elif part_number > self.MAX_PART_ID: raise BadRequest('Part number must be in range 1 - 10000', 'bad_request') file_id = url_match.group(1) bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) part = bucket.upload_part( file_id, part_number, content_length, sha1_sum, input_stream, server_side_encryption ) return part def _assert_account_auth( self, api_url, account_auth_token, account_id, capability, bucket_id=None, file_name=None ): key_sim = self.auth_token_to_key.get(account_auth_token) assert key_sim is not None assert api_url == self.API_URL assert account_id == key_sim.account_id if account_auth_token in self.expired_auth_tokens: raise InvalidAuthToken('auth token expired', 'auth_token_expired') if capability not in key_sim.capabilities: raise Unauthorized('', 'unauthorized') if key_sim.bucket_id_or_none is not None and key_sim.bucket_id_or_none != bucket_id: raise Unauthorized('', 'unauthorized') if key_sim.name_prefix_or_none is not None: if file_name is not None and not file_name.startswith(key_sim.name_prefix_or_none): raise Unauthorized('', 'unauthorized') def _get_bucket_by_id(self, bucket_id) -> BucketSimulator: if bucket_id not in self.bucket_id_to_bucket: raise NonExistentBucket(bucket_id) return self.bucket_id_to_bucket[bucket_id] def _get_bucket_by_name(self, bucket_name): if bucket_name not in self.bucket_name_to_bucket: raise NonExistentBucket(bucket_name) return self.bucket_name_to_bucket[bucket_name] def set_bucket_notification_rules( self, api_url: str, account_auth_token: str, bucket_id: str, rules: Iterable[NotificationRule], ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth( api_url, account_auth_token, bucket.account_id, 'writeBucketNotifications' ) return bucket.set_notification_rules(rules) def get_bucket_notification_rules( self, api_url: str, account_auth_token: str, bucket_id: str ) -> list[NotificationRule]: bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth( api_url, account_auth_token, bucket.account_id, 'readBucketNotifications' ) return bucket.get_notification_rules() b2-sdk-python-2.8.0/b2sdk/_internal/replication/000077500000000000000000000000001474454370000214235ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/replication/__init__.py000066400000000000000000000005251474454370000235360ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/replication/__init__.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/replication/monitoring.py000066400000000000000000000204741474454370000241710ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/replication/monitoring.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import sys from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from queue import Queue from typing import ClassVar, Iterator from ..api import B2Api from ..bucket import Bucket from ..encryption.setting import EncryptionMode from ..file_lock import NO_RETENTION_FILE_SETTING, LegalHold from ..scan.folder import B2Folder from ..scan.path import B2Path from ..scan.policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from ..scan.report import ProgressReport from ..scan.scan import ( AbstractScanReport, AbstractScanResult, CountAndSampleScanReport, zip_folders, ) from .setting import ReplicationRule from .types import ReplicationStatus @dataclass(frozen=True) class ReplicationScanResult(AbstractScanResult): """ Some attributes of source and destination files and their relations which are meaningful for replication monitoring and troubleshooting. Please be aware that only latest file versions are inspected, so any previous file versions are not represented in these results. """ # source attrs source_replication_status: ReplicationStatus | None = None source_has_hide_marker: bool | None = None source_encryption_mode: EncryptionMode | None = None source_has_large_metadata: bool | None = None source_has_file_retention: bool | None = None source_has_legal_hold: bool | None = None # destination attrs destination_replication_status: ReplicationStatus | None = None # source & destination relation attrs metadata_differs: bool | None = None hash_differs: bool | None = None LARGE_METADATA_SIZE: ClassVar[int] = 2048 @classmethod def from_files( cls, source_file: B2Path | None = None, destination_file: B2Path | None = None ) -> ReplicationScanResult: params = {} if source_file: source_file_version = source_file.selected_version params.update( { 'source_replication_status': source_file_version.replication_status, 'source_has_hide_marker': not source_file.is_visible(), 'source_encryption_mode': source_file_version.server_side_encryption.mode, 'source_has_large_metadata': source_file_version.has_large_header, 'source_has_file_retention': source_file_version.file_retention is not NO_RETENTION_FILE_SETTING, 'source_has_legal_hold': source_file_version.legal_hold is LegalHold.ON, } ) if destination_file: params.update( { 'destination_replication_status': destination_file.selected_version.replication_status, } ) if source_file and destination_file: source_version = source_file.selected_version destination_version = destination_file.selected_version params.update( { 'metadata_differs': source_version.file_info != destination_version.file_info, 'hash_differs': (source_version.content_md5 != destination_version.content_md5) or (source_version.content_sha1 != destination_version.content_sha1), } ) return cls(**params) @dataclass class ReplicationReport(CountAndSampleScanReport): SCAN_RESULT_CLASS = ReplicationScanResult @dataclass class ReplicationMonitor: """ Calculates source and (optionally) destination replication statistics. :param b2sdk.v2.Bucket bucket: replication source bucket :param b2sdk.v2.ReplicationRule rule: replication rule to be monitored; should belong to `bucket`'s replication configuration :param b2sdk.v2.B2Api destination_api: B2Api instance for destination bucket; if destination bucket is on the same account as source bucket, omit this parameter and then source bucket's B2Api will be used :param b2sdk.v2.ProgressReport report: instance of ProgressReport which will report scanning progress, by default to stdout :param b2sdk.v2.ScanPoliciesManager scan_policies_manager: a strategy to scan files, so that several files that match some criteria may be omitted :rtype: b2sdk.v2.ReplicationMonitor """ bucket: Bucket rule: ReplicationRule destination_api: B2Api | None = None # if None -> will use `api` of source (bucket) report: ProgressReport = field(default_factory=lambda: ProgressReport(sys.stdout, False)) scan_policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER REPORT_CLASS: ClassVar[AbstractScanReport] = ReplicationReport B2_FOLDER_CLASS: ClassVar[type] = B2Folder QUEUE_SIZE: ClassVar[int] = 20_000 def __post_init__(self): if not self.bucket.replication: raise ValueError(f'Bucket {self.bucket} has no replication configuration') if self.rule not in self.bucket.replication.rules: raise ValueError(f'Rule {self.rule} is not a rule from {self.bucket}') @property def source_api(self) -> B2Api: return self.bucket.api @property def source_folder(self) -> B2_FOLDER_CLASS: return self.B2_FOLDER_CLASS( bucket_name=self.bucket.name, folder_name=self.rule.file_name_prefix, api=self.source_api, ) @property def destination_bucket(self) -> Bucket: destination_api = self.destination_api or self.source_api bucket_id = self.rule.destination_bucket_id return destination_api.get_bucket_by_id(bucket_id) @property def destination_folder(self) -> B2_FOLDER_CLASS: destination_bucket = self.destination_bucket return self.B2_FOLDER_CLASS( bucket_name=destination_bucket.name, folder_name=self.rule.file_name_prefix, api=destination_bucket.api, ) def iter_pairs(self) -> Iterator[tuple[B2Path | None, B2Path | None]]: """ Iterate over files in source and destination and yield pairs. Required for replication inspection in-depth. Return pair of (source B2Path, destination B2Path). Source or destination path may be missing if there's no corresponding destination/source file. """ yield from zip_folders( self.source_folder, self.destination_folder, reporter=self.report, policies_manager=self.scan_policies_manager, ) def scan(self, scan_destination: bool = True) -> AbstractScanReport: """ Scan source bucket (only, or with destination) and return replication report. No destination scan may give limited replication information, since it only checks files on the source bucket without checking whether they we really replicated to destination. It may be handy though if there is no access to replication destination. """ report = self.REPORT_CLASS() queue = Queue(maxsize=self.QUEUE_SIZE) if not scan_destination: def fill_queue(): for path in self.source_folder.all_files( policies_manager=self.scan_policies_manager, reporter=self.report, ): queue.put((path,), block=True) queue.put(None, block=True) else: def fill_queue(): for pair in self.iter_pairs(): queue.put(pair, block=True) queue.put(None, block=True) def consume_queue(): while True: items = queue.get(block=True) if items is None: # using None as "end of queue" marker break report.add(*items) with ThreadPoolExecutor(max_workers=2) as thread_pool: futures = [ thread_pool.submit(fill_queue), thread_pool.submit(consume_queue), ] for future in futures: future.result() return report b2-sdk-python-2.8.0/b2sdk/_internal/replication/setting.py000066400000000000000000000162601474454370000234570ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/replication/setting.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import re from builtins import classmethod from dataclasses import dataclass, field from typing import ClassVar @dataclass class ReplicationRule: """ Hold information about replication rule: destination bucket, priority, prefix and rule name. """ DEFAULT_PRIORITY: ClassVar[int] = 128 destination_bucket_id: str name: str file_name_prefix: str = '' is_enabled: bool = True priority: int = DEFAULT_PRIORITY include_existing_files: bool = False REPLICATION_RULE_REGEX: ClassVar = re.compile(r'^[a-zA-Z0-9_\-]{1,64}$') MIN_PRIORITY: ClassVar[int] = 1 MAX_PRIORITY: ClassVar[int] = 2**31 - 1 def __post_init__(self): if not self.destination_bucket_id: raise ValueError('destination_bucket_id is required') if not self.REPLICATION_RULE_REGEX.match(self.name): raise ValueError('replication rule name is invalid') if not (self.MIN_PRIORITY <= self.priority <= self.MAX_PRIORITY): raise ValueError( 'priority should be within [%d, %d] interval' % ( self.MIN_PRIORITY, self.MAX_PRIORITY, ) ) def as_dict(self) -> dict: return { 'destinationBucketId': self.destination_bucket_id, 'fileNamePrefix': self.file_name_prefix, 'includeExistingFiles': self.include_existing_files, 'isEnabled': self.is_enabled, 'priority': self.priority, 'replicationRuleName': self.name, } @classmethod def from_dict(cls, value_dict: dict) -> ReplicationRule: kwargs = {} for field_, protocolField in ( ('destination_bucket_id', 'destinationBucketId'), ('name', 'replicationRuleName'), ('file_name_prefix', 'fileNamePrefix'), ('include_existing_files', 'includeExistingFiles'), ('is_enabled', 'isEnabled'), ('priority', 'priority'), ): value = value_dict.get( protocolField ) # refactor to := when dropping Python 3.7, maybe even dict expression if value is not None: kwargs[field_] = value return cls(**kwargs) @dataclass class ReplicationConfiguration: """ Hold information about bucket replication configuration """ # configuration as source: rules: list[ReplicationRule] = field(default_factory=list) source_key_id: str | None = None # configuration as destination: source_to_destination_key_mapping: dict[str, str] = field(default_factory=dict) def __post_init__(self): if self.rules and not self.source_key_id: raise ValueError('source_key_id must not be empty') for source, destination in self.source_to_destination_key_mapping.items(): if not source or not destination: raise ValueError( f'source_to_destination_key_mapping must not contain \ empty keys or values: ({source}, {destination})' ) @property def is_source(self) -> bool: return bool(self.source_key_id) def get_source_configuration_as_dict(self) -> dict: return { 'rules': self.rules, 'source_key_id': self.source_key_id, } @property def is_destination(self) -> bool: return bool(self.source_to_destination_key_mapping) def get_destination_configuration_as_dict(self) -> dict: return { 'source_to_destination_key_mapping': self.source_to_destination_key_mapping, } def as_dict(self) -> dict: """ Represent the setting as a dict, for example: .. code-block:: python { "asReplicationSource": { "replicationRules": [ { "destinationBucketId": "c5f35d53a90a7ea284fb0719", "fileNamePrefix": "", "includeExistingFiles": True, "isEnabled": true, "priority": 1, "replicationRuleName": "replication-us-west" }, { "destinationBucketId": "55f34d53a96a7ea284fb0719", "fileNamePrefix": "", "includeExistingFiles": True, "isEnabled": true, "priority": 2, "replicationRuleName": "replication-us-west-2" } ], "sourceApplicationKeyId": "10053d55ae26b790000000006" }, "asReplicationDestination": { "sourceToDestinationKeyMapping": { "10053d55ae26b790000000045": "10053d55ae26b790000000004", "10053d55ae26b790000000046": "10053d55ae26b790030000004" } } } """ result = { 'asReplicationSource': { 'replicationRules': [rule.as_dict() for rule in self.rules], 'sourceApplicationKeyId': self.source_key_id, } if self.is_source else None, 'asReplicationDestination': { 'sourceToDestinationKeyMapping': self.source_to_destination_key_mapping, } if self.is_destination else None, } return result serialize_to_json_for_request = as_dict @classmethod def from_dict(cls, value_dict: dict) -> ReplicationConfiguration: source_dict = value_dict.get('asReplicationSource') or {} destination_dict = value_dict.get('asReplicationDestination') or {} return cls( rules=[ ReplicationRule.from_dict(rule_dict) for rule_dict in source_dict.get('replicationRules', []) ], source_key_id=source_dict.get('sourceApplicationKeyId'), source_to_destination_key_mapping=destination_dict.get('sourceToDestinationKeyMapping') or {}, ) @dataclass class ReplicationConfigurationFactory: is_client_authorized_to_read: bool value: ReplicationConfiguration | None @classmethod def from_bucket_dict(cls, bucket_dict: dict) -> ReplicationConfigurationFactory: """ Returns ReplicationConfigurationFactory for the given bucket dict retrieved from the api. """ replication_dict = bucket_dict.get('replicationConfiguration') or {} value_dict = replication_dict.get('value') or {} return cls( is_client_authorized_to_read=replication_dict.get('isClientAuthorizedToRead', True), value=ReplicationConfiguration.from_dict(value_dict), ) b2-sdk-python-2.8.0/b2sdk/_internal/replication/setup.py000066400000000000000000000320021474454370000231320ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/replication/setup.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import itertools import logging # b2 replication-setup [--profile profileName] --destination-profile destinationProfileName sourceBucketPath destinationBucketName [ruleName] # b2 replication-debug [--profile profileName] [--destination-profile destinationProfileName] bucketPath # b2 replication-status [--profile profileName] [--destination-profile destinationProfileName] [sourceBucketPath] [destinationBucketPath] # b2 replication-pause [--profile profileName] (sourceBucketName|sourceBucketPath) [replicationRuleName] # b2 replication-unpause [--profile profileName] (sourceBucketName|sourceBucketPath) [replicationRuleName] # b2 replication-accept destinationBucketName sourceKeyId [destinationKeyId] # b2 replication-deny destinationBucketName sourceKeyId from collections.abc import Iterable from typing import ClassVar from b2sdk._internal.api import B2Api from b2sdk._internal.application_key import ApplicationKey from b2sdk._internal.bucket import Bucket from b2sdk._internal.replication.setting import ReplicationConfiguration, ReplicationRule from b2sdk._internal.utils import B2TraceMeta logger = logging.getLogger(__name__) class ReplicationSetupHelper(metaclass=B2TraceMeta): """class with various methods that help with setting up repliction""" PRIORITY_OFFSET: ClassVar[int] = 5 #: how far to to put the new rule from the existing rules DEFAULT_PRIORITY: ClassVar[int] = ( ReplicationRule.DEFAULT_PRIORITY ) #: what priority to set if there are no preexisting rules MAX_PRIORITY: ClassVar[int] = ( ReplicationRule.MAX_PRIORITY ) #: maximum allowed priority of a replication rule DEFAULT_SOURCE_CAPABILITIES: ClassVar[tuple[str, ...]] = ( 'readFiles', 'readFileLegalHolds', 'readFileRetentions', ) DEFAULT_DESTINATION_CAPABILITIES: ClassVar[tuple[str, ...]] = ( 'writeFiles', 'writeFileLegalHolds', 'writeFileRetentions', 'deleteFiles', ) def setup_both( self, source_bucket: Bucket, destination_bucket: Bucket, name: str | None = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule prefix: str | None = None, include_existing_files: bool = False, ) -> tuple[Bucket, Bucket]: # setup source key source_key = self._get_source_key( source_bucket, prefix, source_bucket.replication, ) # setup destination new_destination_bucket = self.setup_destination( source_key.id_, destination_bucket, ) # setup source new_source_bucket = self.setup_source( source_bucket, source_key, destination_bucket, prefix, name, priority, include_existing_files, ) return new_source_bucket, new_destination_bucket def setup_destination( self, source_key_id: str, destination_bucket: Bucket, ) -> Bucket: api: B2Api = destination_bucket.api source_configuration = ( destination_bucket.replication.get_source_configuration_as_dict() if destination_bucket.replication else {} ) destination_configuration = ( destination_bucket.replication.get_destination_configuration_as_dict() if destination_bucket.replication else {'source_to_destination_key_mapping': {}} ) keys_to_purge, destination_key = self._get_destination_key( api, destination_bucket, ) # note: no clean up of keys_to_purge is actually done destination_configuration['source_to_destination_key_mapping'][source_key_id] = ( destination_key.id_ ) new_replication_configuration = ReplicationConfiguration( **source_configuration, **destination_configuration, ) return destination_bucket.update( if_revision_is=destination_bucket.revision, replication=new_replication_configuration, ) @classmethod def _get_destination_key( cls, api: B2Api, destination_bucket: Bucket, ): keys_to_purge = [] if destination_bucket.replication is not None: current_destination_key_ids = ( destination_bucket.replication.source_to_destination_key_mapping.values() ) else: current_destination_key_ids = [] key = None for current_destination_key_id in current_destination_key_ids: # potential inefficiency here as we are fetching keys one by one, however # the number of keys on an account is limited to a 100 000 000 per account lifecycle # while the number of keys in the map can be expected to be very low current_destination_key = api.get_key(current_destination_key_id) if current_destination_key is None: logger.debug( 'zombie key found in replication destination_configuration.source_to_destination_key_mapping: %s', current_destination_key_id, ) keys_to_purge.append(current_destination_key_id) continue if ( current_destination_key.has_capabilities(cls.DEFAULT_DESTINATION_CAPABILITIES) and not current_destination_key.name_prefix ): logger.debug('matching destination key found: %s', current_destination_key_id) key = current_destination_key # not breaking here since we want to fill the purge list else: logger.info('non-matching destination key found: %s', current_destination_key) if not key: logger.debug('no matching key found, making a new one') key = cls._create_destination_key( name=destination_bucket.name[:91] + '-replidst', bucket=destination_bucket, prefix=None, ) return keys_to_purge, key def setup_source( self, source_bucket: Bucket, source_key: ApplicationKey, destination_bucket: Bucket, prefix: str | None = None, name: str | None = None, #: name for the new replication rule priority: int = None, #: priority for the new replication rule include_existing_files: bool = False, ) -> Bucket: if prefix is None: prefix = '' if source_bucket.replication: current_source_rules = source_bucket.replication.rules destination_configuration = ( source_bucket.replication.get_destination_configuration_as_dict() ) else: current_source_rules = [] destination_configuration = {} priority = self._get_priority_for_new_rule( current_source_rules, priority, ) name = self._get_new_rule_name( current_source_rules, destination_bucket, name, ) new_rr = ReplicationRule( name=name, priority=priority, destination_bucket_id=destination_bucket.id_, file_name_prefix=prefix, include_existing_files=include_existing_files, ) new_replication_configuration = ReplicationConfiguration( source_key_id=source_key.id_, rules=current_source_rules + [new_rr], **destination_configuration, ) return source_bucket.update( if_revision_is=source_bucket.revision, replication=new_replication_configuration, ) @classmethod def _get_source_key( cls, source_bucket: Bucket, prefix: str, current_replication_configuration: ReplicationConfiguration, ) -> ApplicationKey: api = source_bucket.api if current_replication_configuration is not None: current_source_key = api.get_key(current_replication_configuration.source_key_id) do_create_key = cls._should_make_new_source_key( current_replication_configuration, current_source_key, ) if not do_create_key: return current_source_key new_key = cls._create_source_key( name=source_bucket.name[:91] + '-replisrc', bucket=source_bucket, prefix=prefix, ) return new_key @classmethod def _should_make_new_source_key( cls, current_replication_configuration: ReplicationConfiguration, current_source_key: ApplicationKey | None, ) -> bool: if current_replication_configuration.source_key_id is None: logger.debug('will create a new source key because no key is set') return True if current_source_key is None: logger.debug( 'will create a new source key because current key "%s" was deleted', current_replication_configuration.source_key_id, ) return True if current_source_key.name_prefix: logger.debug( 'will create a new source key because current key %s had a prefix: "%s"', current_source_key.name_prefix, ) return True if not current_source_key.has_capabilities(cls.DEFAULT_SOURCE_CAPABILITIES): logger.debug( 'will create a new source key because %s installed so far does not have enough permissions for replication source: ', current_source_key.id_, current_source_key.capabilities, ) return True return False # current key is ok @classmethod def _create_source_key( cls, name: str, bucket: Bucket, prefix: str | None = None, ) -> ApplicationKey: # in this implementation we ignore the prefix and create a full key, because # if someone would need a different (wider) key later, all replication # destinations would have to start using new keys and it's not feasible # from organizational perspective, while the prefix of uploaded files can be # restricted on the rule level prefix = None capabilities = cls.DEFAULT_SOURCE_CAPABILITIES return cls._create_key(name, bucket, prefix, capabilities) @classmethod def _create_destination_key( cls, name: str, bucket: Bucket, prefix: str | None = None, ) -> ApplicationKey: capabilities = cls.DEFAULT_DESTINATION_CAPABILITIES return cls._create_key(name, bucket, prefix, capabilities) @classmethod def _create_key( cls, name: str, bucket: Bucket, prefix: str | None = None, capabilities=tuple(), ) -> ApplicationKey: api: B2Api = bucket.api return api.create_key( capabilities=capabilities, key_name=name, bucket_id=bucket.id_, name_prefix=prefix, ) @classmethod def _get_priority_for_new_rule( cls, current_rules: Iterable[ReplicationRule], priority: int | None = None, ): if priority is not None: return priority if current_rules: # ignore a case where the existing rrs need to have their priorities decreased to make space (max is 2**31-1) existing_priority = max(rr.priority for rr in current_rules) return min(existing_priority + cls.PRIORITY_OFFSET, cls.MAX_PRIORITY) return cls.DEFAULT_PRIORITY @classmethod def _get_new_rule_name( cls, current_rules: Iterable[ReplicationRule], destination_bucket: Bucket, name: str | None = None, ): if name is not None: return name existing_names = set(rr.name for rr in current_rules) suffixes = cls._get_rule_name_candidate_suffixes() while True: candidate = f'{destination_bucket.name}{next(suffixes)}' # use := after dropping 3.7 if candidate not in existing_names: return candidate @classmethod def _get_rule_name_candidate_suffixes(cls): """ >>> a = ReplicationSetupHelper._get_rule_name_candidate_suffixes() >>> [next(a) for i in range(10)] ['', '2', '3', '4', '5', '6', '7', '8', '9', '10'] """ return map(str, itertools.chain([''], itertools.count(2))) @classmethod def _partion_bucket_path(cls, bucket_path: str) -> tuple[str, str]: bucket_name, _, path = bucket_path.partition('/') return bucket_name, path b2-sdk-python-2.8.0/b2sdk/_internal/replication/types.py000066400000000000000000000013061474454370000231410ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/replication/types.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from enum import Enum, unique @unique class ReplicationStatus(Enum): PENDING = 'PENDING' COMPLETED = 'COMPLETED' FAILED = 'FAILED' REPLICA = 'REPLICA' @classmethod def from_response_headers(cls, headers: dict) -> ReplicationStatus | None: value = headers.get('X-Bz-Replication-Status', None) return value and cls[value.upper()] b2-sdk-python-2.8.0/b2sdk/_internal/requests/000077500000000000000000000000001474454370000207655ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/requests/LICENSE000066400000000000000000000236351474454370000220030ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.b2-sdk-python-2.8.0/b2sdk/_internal/requests/NOTICE000066400000000000000000000004411474454370000216700ustar00rootroot00000000000000Requests Copyright 2019 Kenneth Reitz Copyright 2021 Backblaze Inc. Changes made to the original source: requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` in order to NOT decompress data based on Content-Encoding headerb2-sdk-python-2.8.0/b2sdk/_internal/requests/README.md000066400000000000000000000003501474454370000222420ustar00rootroot00000000000000This module contains modified parts of the requests module (https://github.com/psf/requests). The modules original license is included in LICENSE. Changes made to the original source are listed in NOTICE, along with original NOTICE.b2-sdk-python-2.8.0/b2sdk/_internal/requests/__init__.py000066400000000000000000000063131474454370000231010ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/requests/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # Copyright 2019 Kenneth Reitz # # License https://www.backblaze.com/using_b2_code.html # License Apache License 2.0 (http://www.apache.org/licenses/ and LICENSE file in this directory) # ###################################################################### """This file contains modified parts of the requests module (https://github.com/psf/requests, models.py), original Copyright 2019 Kenneth Reitz Changes made to the original source: see NOTICE """ from requests import Response, ConnectionError from requests.exceptions import ChunkedEncodingError, ContentDecodingError, StreamConsumedError from requests.utils import iter_slices, stream_decode_response_unicode from urllib3.exceptions import ProtocolError, DecodeError, ReadTimeoutError from . import included_source_meta class NotDecompressingResponse(Response): def iter_content(self, chunk_size=1, decode_unicode=False): def generate(): # Special case for urllib3. if hasattr(self.raw, 'stream'): try: # set decode_content to False to prevent decompressing files that # when Content-Encoding response header is set for chunk in self.raw.stream(chunk_size, decode_content=False): yield chunk except ProtocolError as e: raise ChunkedEncodingError(e) except DecodeError as e: raise ContentDecodingError(e) except ReadTimeoutError as e: raise ConnectionError(e) else: # Standard file-like object. while True: chunk = self.raw.read(chunk_size) if not chunk: break yield chunk self._content_consumed = True if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() elif chunk_size is not None and not isinstance(chunk_size, int): raise TypeError('chunk_size must be an int, it is instead a %s.' % type(chunk_size)) # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) stream_chunks = generate() chunks = reused_chunks if self._content_consumed else stream_chunks if decode_unicode: chunks = stream_decode_response_unicode(chunks, self) return chunks @classmethod def from_builtin_response(cls, response: Response): """ Create a :class:`b2sdk._internal.requests.NotDecompressingResponse` object from a :class:`requests.Response` object. Don't use :code:`Response.__getstate__` and :code:`Response.__setstate__` because these assume that the content has been consumed, which will never be true in our case. """ new_response = cls() for attr_name in cls.__attrs__: setattr(new_response, attr_name, getattr(response, attr_name)) new_response.raw = response.raw return new_response b2-sdk-python-2.8.0/b2sdk/_internal/requests/included_source_meta.py000066400000000000000000000015411474454370000255150ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/requests/included_source_meta.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from b2sdk._internal.included_sources import IncludedSourceMeta, add_included_source included_source_meta = IncludedSourceMeta( 'requests', 'Included in a revised form', { 'NOTICE': """Requests Copyright 2019 Kenneth Reitz Copyright 2021 Backblaze Inc. Changes made to the original source: requests.models.Response.iter_content has been overridden to pass `decode_content=False` argument to `self.raw.stream` in order to NOT decompress data based on Content-Encoding header""" }, ) add_included_source(included_source_meta) b2-sdk-python-2.8.0/b2sdk/_internal/scan/000077500000000000000000000000001474454370000200365ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/scan/__init__.py000066400000000000000000000005161474454370000221510ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/scan/__init__.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/scan/exception.py000066400000000000000000000053761474454370000224210ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/scan/exception.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from contextlib import contextmanager from typing import Iterator from ..exception import B2Error, B2SimpleError class EnvironmentEncodingError(B2Error): """ Raised when a file name can not be decoded with system encoding. """ def __init__(self, filename, encoding): """ :param filename: an encoded file name :type filename: str, bytes :param str encoding: file name encoding """ super().__init__() self.filename = filename self.encoding = encoding def __str__(self): return f"""file name {self.filename} cannot be decoded with system encoding ({self.encoding}). We think this is an environment error which you should workaround by setting your system encoding properly, for example like this: export LANG=en_US.UTF-8""" class InvalidArgument(B2Error): """ Raised when one or more arguments are invalid """ def __init__(self, parameter_name, message): """ :param parameter_name: name of the function argument :param message: brief explanation of misconfiguration """ super().__init__() self.parameter_name = parameter_name self.message = message def __str__(self): return f'{self.parameter_name} {self.message}' class UnsupportedFilename(B2Error): """ Raised when a filename is not supported by the scan operation """ def __init__(self, message, filename): """ :param message: brief explanation of why the filename was not supported :param filename: name of the file which is not supported """ super().__init__() self.filename = filename self.message = message def __str__(self): return f'{self.message}: {self.filename}' @contextmanager def check_invalid_argument( parameter_name: str, message: str, *exceptions: type[Exception] ) -> Iterator[None]: """Raise `InvalidArgument` in case of one of given exception was thrown.""" try: yield except exceptions as exc: if not message: message = str(exc) raise InvalidArgument(parameter_name, message) from exc class BaseDirectoryError(B2SimpleError): def __init__(self, path): self.path = path super().__init__(path) class EmptyDirectory(BaseDirectoryError): pass class UnableToCreateDirectory(BaseDirectoryError): pass class NotADirectory(BaseDirectoryError): pass b2-sdk-python-2.8.0/b2sdk/_internal/scan/folder.py000066400000000000000000000374241474454370000216750ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/scan/folder.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging import os import platform import re import sys from abc import ABCMeta, abstractmethod from pathlib import Path from typing import Iterator from ..utils import fix_windows_path_limit, get_file_mtime, validate_b2_file_name from .exception import ( EmptyDirectory, EnvironmentEncodingError, NotADirectory, UnableToCreateDirectory, UnsupportedFilename, ) from .path import AbstractPath, B2Path, LocalPath from .policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from .report import ProgressReport DRIVE_MATCHER = re.compile(r'^([A-Za-z]):([/\\])') ABSOLUTE_PATH_MATCHER = re.compile(r'^(/)|^(\\)') RELATIVE_PATH_MATCHER = re.compile( # "abc" and "xyz" represent anything, including "nothing" r'^(\.\.[/\\])|' # ../abc or ..\abc + r'^(\.[/\\])|' # ./abc or .\abc + r'([/\\]\.\.[/\\])|' # abc/../xyz or abc\..\xyz or abc\../xyz or abc/..\xyz + r'([/\\]\.[/\\])|' # abc/./xyz or abc\.\xyz or abc\./xyz or abc/.\xyz + r'([/\\]\.\.)$|' # abc/.. or abc\.. + r'([/\\]\.)$|' # abc/. or abc\. + r'^(\.\.)$|' # just ".." + r'([/\\][/\\])|' # abc\/xyz or abc/\xyz or abc//xyz or abc\\xyz + r'^(\.)$' # just "." ) logger = logging.getLogger(__name__) class AbstractFolder(metaclass=ABCMeta): """ Interface to a folder full of files, which might be a B2 bucket, a virtual folder in a B2 bucket, or a directory on a local file system. Files in B2 may have multiple versions, while files in local folders have just one. """ @abstractmethod def all_files( self, reporter: ProgressReport | None, policies_manager=DEFAULT_SCAN_MANAGER ) -> Iterator[AbstractPath]: """ Return an iterator over all of the files in the folder, in the order that B2 uses (lexicographic by object path). It also performs filtering using policies manager. No matter what the folder separator on the local file system is, "/" is used in the returned file names. If a file is found, but does not exist (for example due to a broken symlink or a race), reporter will be informed about each such problem. :param reporter: a place to report errors :param policies_manager: a policies manager object """ @abstractmethod def folder_type(self): """ Return one of: 'b2', 'local'. :rtype: str """ @abstractmethod def make_full_path(self, file_name): """ Return the full path to the file. :param file_name: a file name :type file_name: str :rtype: str """ def join_b2_path(relative_dir_path: str | Path, file_name: str): """ Like os.path.join, but for B2 file names where the root directory is called ''. """ relative_dir_path = str(relative_dir_path) if relative_dir_path in ('', '.'): return file_name else: return relative_dir_path + '/' + file_name if sys.platform == 'win32': def _file_read_access(path): try: with open(path, 'rb', buffering=0): return True except (FileNotFoundError, PermissionError): return False else: def _file_read_access(path): return os.access(path, os.R_OK) class LocalFolder(AbstractFolder): """ Folder interface to a directory on the local machine. """ def __init__(self, root: str | Path): """ Initialize a new folder. :param root: path to the root of the local folder. Must be unicode. """ if isinstance(root, Path): root = str(root) if not isinstance(root, str): raise ValueError('folder path should be str or pathlib.Path: %s' % repr(root)) self.root = fix_windows_path_limit(os.path.abspath(root)) def folder_type(self): """ Return folder type. :rtype: str """ return 'local' def all_files( self, reporter: ProgressReport | None, policies_manager=DEFAULT_SCAN_MANAGER ) -> Iterator[LocalPath]: """ Yield all files. Yield a File object for each of the files anywhere under this folder, in the order they would appear in B2, unless the path is excluded by policies manager. :param reporter: a place to report errors :param policies_manager: a policy manager object, default is DEFAULT_SCAN_MANAGER :return: an iterator over all files in the folder in the order they would appear in B2 """ root_path = Path(self.root) local_paths = self._walk_relative_paths(root_path, Path(''), reporter, policies_manager) # Crucial to return the "correct" order of the files yield from sorted(local_paths, key=lambda lp: lp.relative_path) def make_full_path(self, file_name): """ Convert a file name into an absolute path, ensure it is not outside self.root :param file_name: a file name :type file_name: str """ # Fix OS path separators file_name = file_name.replace('/', os.path.sep) # Generate the full path to the file full_path = os.path.normpath(os.path.join(self.root, file_name)) # Get the common prefix between the new full_path and self.root common_prefix = os.path.commonprefix([full_path, self.root]) # Ensure the new full_path is inside the self.root directory if common_prefix != self.root: raise UnsupportedFilename('illegal file name', full_path) return full_path def ensure_present(self): """ Make sure that the directory exists. """ if not os.path.exists(self.root): try: os.mkdir(self.root) except OSError: raise UnableToCreateDirectory(self.root) elif not os.path.isdir(self.root): raise NotADirectory(self.root) def ensure_non_empty(self): """ Make sure that the directory exists and is non-empty. """ self.ensure_present() if not os.listdir(self.root): raise EmptyDirectory(self.root) def _walk_relative_paths( self, local_dir: Path, relative_dir_path: Path, reporter: ProgressReport, policies_manager: ScanPoliciesManager, visited_symlinks: set[int] | None = None, ): """ Yield a File object for each of the files anywhere under this folder, unless the path is excluded by policies manager. :param local_dir: the path to the local directory that we are currently inspecting :param relative_dir_path: the path of this dir relative to the scan point, or Path('') if at scan point :param reporter: a reporter object to report errors and warnings :param policies_manager: a policies manager object :param visited_symlinks: a set of paths to symlinks that have already been visited. Using inode numbers to reduce memory usage """ # Collect the names. We do this before returning any results, because # directories need to sort as if their names end in '/'. # # With a directory containing 'a', 'a.txt', and 'a0.txt', with 'a' being # a directory containing 'b.txt', and 'c.txt', the results returned # should be: # # a.txt # a/b.txt # a/c.txt # a0.txt # # This is because in Unicode '.' comes before '/', which comes before '0'. visited_symlinks = visited_symlinks or set() if local_dir.is_symlink(): inode_number = local_dir.resolve().stat().st_ino if inode_number in visited_symlinks: if reporter: reporter.circular_symlink_skipped(str(local_dir)) return # Skip if symlink already visited visited_symlinks.add(inode_number) try: dir_children = sorted(local_dir.iterdir()) except PermissionError: # `chmod -r dir` can trigger this if reporter is not None: reporter.local_permission_error(str(local_dir)) return for local_path in dir_children: name = local_path.name relative_file_path = join_b2_path(relative_dir_path, name) if policies_manager.exclude_all_symlinks and local_path.is_symlink(): if reporter is not None: reporter.symlink_skipped(str(local_path)) continue try: validate_b2_file_name(name) except ValueError as e: if reporter is not None: reporter.invalid_name(str(local_path), str(e)) continue try: is_dir = local_path.is_dir() except PermissionError: # `chmod -x dir` can trigger this if reporter is not None and not policies_manager.should_exclude_local_directory( str(relative_file_path) ): reporter.local_permission_error(str(local_path)) continue if is_dir: if policies_manager.should_exclude_local_directory(str(relative_file_path)): continue # Skip excluded directories # Recurse into directories yield from self._walk_relative_paths( local_path, relative_file_path, reporter, policies_manager, visited_symlinks ) else: if policies_manager.should_exclude_relative_path(relative_file_path): continue # Skip excluded files try: file_mod_time = get_file_mtime(str(local_path)) file_size = local_path.stat().st_size except OSError: if reporter is not None: reporter.local_access_error(str(local_path)) continue local_scan_path = LocalPath( absolute_path=self.make_full_path(str(relative_file_path)), relative_path=str(relative_file_path), mod_time=file_mod_time, size=file_size, ) if policies_manager.should_exclude_local_path(local_scan_path): continue # Skip excluded files if not _file_read_access(local_path): if reporter is not None: reporter.local_permission_error(str(local_path)) continue yield local_scan_path @classmethod def _handle_non_unicode_file_name(cls, name): """ Decide what to do with a name returned from os.listdir() that isn't unicode. We think that this only happens when the file name can't be decoded using the file system encoding. Just in case that's not true, we'll allow all-ascii names. """ # if it's all ascii, allow it if all(b <= 127 for b in name): return name raise EnvironmentEncodingError(repr(name), sys.getfilesystemencoding()) def __repr__(self): return f'LocalFolder({self.root})' def b2_parent_dir(file_name): # Various Parent dir getting method have been tested, and this one seems to be the faste # After dropping python 3.9 support: refactor this use the "match" syntax try: dir_name, _ = file_name.rsplit('/', 1) except ValueError: return '' return dir_name class B2Folder(AbstractFolder): """ Folder interface to b2. """ def __init__(self, bucket_name, folder_name, api): """ :param bucket_name: a name of the bucket :type bucket_name: str :param folder_name: a folder name :type folder_name: str :param api: an API object :type api: b2sdk._internal.api.B2Api """ self.bucket_name = bucket_name self.folder_name = folder_name self.bucket = api.get_bucket_by_name(bucket_name) self.api = api self.prefix = self.folder_name if self.prefix and self.prefix[-1] != '/': self.prefix += '/' def all_files( self, reporter: ProgressReport | None, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER, ) -> Iterator[B2Path]: """ Yield all files. """ current_name = None last_ignored_dir = None current_versions = [] current_file_version = None for file_version in self.get_file_versions(): if current_file_version is None: current_file_version = file_version assert file_version.file_name.startswith(self.prefix) if file_version.action == 'start': continue file_name = file_version.file_name[len(self.prefix) :] if last_ignored_dir is not None and file_name.startswith(last_ignored_dir): continue dir_name = b2_parent_dir(file_name) if policies_manager.should_exclude_b2_directory(dir_name): last_ignored_dir = dir_name + '/' continue else: last_ignored_dir = None if policies_manager.should_exclude_b2_file_version(file_version, file_name): continue self._validate_file_name(file_name) if current_name != file_name and current_name is not None and current_versions: yield B2Path( relative_path=current_name, selected_version=current_versions[0], all_versions=current_versions, ) current_versions = [] current_name = file_name current_versions.append(file_version) if current_name is not None and current_versions: yield B2Path( relative_path=current_name, selected_version=current_versions[0], all_versions=current_versions, ) def get_file_versions(self): for file_version, _ in self.bucket.ls( self.folder_name, latest_only=False, recursive=True, ): yield file_version def _validate_file_name(self, file_name): # Do not allow relative paths in file names if RELATIVE_PATH_MATCHER.search(file_name): raise UnsupportedFilename( 'scan does not support file names that include relative paths', file_name ) # Do not allow absolute paths in file names if ABSOLUTE_PATH_MATCHER.search(file_name): raise UnsupportedFilename( 'scan does not support file names with absolute paths', file_name ) # On Windows, do not allow drive letters in file names if platform.system() == 'Windows' and DRIVE_MATCHER.search(file_name): raise UnsupportedFilename( 'scan does not support file names with drive letters', file_name ) def folder_type(self): """ Return folder type. :rtype: str """ return 'b2' def make_full_path(self, file_name): """ Make an absolute path from a file name. :param file_name: a file name :type file_name: str """ if self.folder_name == '': return file_name else: return self.folder_name + '/' + file_name def __str__(self): return f'B2Folder({self.bucket_name}, {self.folder_name})' b2-sdk-python-2.8.0/b2sdk/_internal/scan/folder_parser.py000066400000000000000000000037451474454370000232500ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/scan/folder_parser.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from .exception import InvalidArgument from .folder import B2Folder, LocalFolder def parse_folder(folder_name, api, local_folder_class=LocalFolder, b2_folder_class=B2Folder): """ Take either a local path, or a B2 path, and returns a Folder object for it. B2 paths look like: b2://bucketName/path/name. The '//' is optional. Anything else is treated like a local folder. :param folder_name: a name of the folder, either local or remote :type folder_name: str :param api: an API object :type api: :class:`~b2sdk.v2.B2Api` :param local_folder_class: class to handle local folders :type local_folder_class: `~b2sdk.v2.AbstractFolder` :param b2_folder_class: class to handle B2 folders :type b2_folder_class: `~b2sdk.v2.AbstractFolder` """ if folder_name.startswith('b2://'): return _parse_bucket_and_folder(folder_name[5:], api, b2_folder_class) elif folder_name.startswith('b2:') and folder_name[3].isalnum(): return _parse_bucket_and_folder(folder_name[3:], api, b2_folder_class) else: return local_folder_class(folder_name) def _parse_bucket_and_folder(bucket_and_path, api, b2_folder_class): """ Turn 'my-bucket/foo' into B2Folder(my-bucket, foo). """ if '//' in bucket_and_path: raise InvalidArgument('folder_name', "'//' not allowed in path names") if '/' not in bucket_and_path: bucket_name = bucket_and_path folder_name = '' else: (bucket_name, folder_name) = bucket_and_path.split('/', 1) if folder_name.endswith('/'): folder_name = folder_name[:-1] return b2_folder_class(bucket_name, folder_name, api) b2-sdk-python-2.8.0/b2sdk/_internal/scan/path.py000066400000000000000000000052451474454370000213520ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/scan/path.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from abc import ABC, abstractmethod from ..file_version import FileVersion class AbstractPath(ABC): """ Represent a path in a source or destination folder - be it B2 or local """ def __init__(self, relative_path: str, mod_time: int, size: int): self.relative_path = relative_path self.mod_time = mod_time self.size = size @abstractmethod def is_visible(self) -> bool: """Is the path visible/not deleted on it's storage""" def __repr__(self): return f'{self.__class__.__name__}({repr(self.relative_path)}, {repr(self.mod_time)}, {repr(self.size)})' class LocalPath(AbstractPath): __slots__ = ['absolute_path', 'relative_path', 'mod_time', 'size'] def __init__(self, absolute_path: str, relative_path: str, mod_time: int, size: int): self.absolute_path = absolute_path super().__init__(relative_path, mod_time, size) def is_visible(self) -> bool: return True def __eq__(self, other): return ( self.absolute_path == other.absolute_path and self.relative_path == other.relative_path and self.mod_time == other.mod_time and self.size == other.size ) class B2Path(AbstractPath): __slots__ = ['relative_path', 'selected_version', 'all_versions'] def __init__( self, relative_path: str, selected_version: FileVersion, all_versions: list[FileVersion] ): self.selected_version = selected_version self.all_versions = all_versions self.relative_path = relative_path def is_visible(self) -> bool: return self.selected_version.action != 'hide' @property def mod_time(self) -> int: return self.selected_version.mod_time_millis @property def size(self) -> int: return self.selected_version.size def __repr__(self): return '{}({}, [{}])'.format( self.__class__.__name__, self.relative_path, ', '.join( f'({repr(fv.id_)}, {repr(fv.mod_time_millis)}, {repr(fv.action)})' for fv in self.all_versions ), ) def __eq__(self, other): return ( self.relative_path == other.relative_path and self.selected_version == other.selected_version and self.all_versions == other.all_versions ) b2-sdk-python-2.8.0/b2sdk/_internal/scan/policies.py000066400000000000000000000215421474454370000222230ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/scan/policies.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging import re from typing import Iterable from ..file_version import FileVersion from .exception import InvalidArgument, check_invalid_argument from .path import LocalPath logger = logging.getLogger(__name__) class RegexSet: """ Hold a (possibly empty) set of regular expressions and know how to check whether a string matches any of them. """ def __init__(self, regex_iterable): """ :param regex_iterable: an interable which yields regexes """ self._compiled_list = [re.compile(r) for r in regex_iterable] def matches(self, s): """ Check whether a string matches any of regular expressions. :param s: a string to check :type s: str :rtype: bool """ return any(c.match(s) is not None for c in self._compiled_list) def convert_dir_regex_to_dir_prefix_regex(dir_regex: str | re.Pattern) -> str: """ The patterns used to match directory names (and file names) are allowed to match a prefix of the name. This 'feature' was unintentional, but is being retained for compatibility. This means that a regex that matches a directory name can't be used directly to match against a file name and test whether the file should be excluded because it matches the directory. The pattern 'photos' will match directory names 'photos' and 'photos2', and should exclude files 'photos/kitten.jpg', and 'photos2/puppy.jpg'. It should not exclude 'photos.txt', because there is no directory name that matches. On the other hand, the pattern 'photos$' should match 'photos/kitten.jpg', but not 'photos2/puppy.jpg', nor 'photos.txt' If the original regex is valid, there are only two cases to consider: either the regex ends in '$' or does not. :param dir_regex: a regular expression string or literal :return: a regular expression string which matches the directory prefix """ if isinstance(dir_regex, re.Pattern): dir_regex = dir_regex.pattern if dir_regex.endswith('$'): return dir_regex[:-1] + r'/' else: return dir_regex + r'.*?/' class IntegerRange: """ Hold a range of two integers. If the range value is None, it indicates that the value should be treated as -Inf (for begin) or +Inf (for end). """ def __init__(self, begin, end): """ :param begin: begin position of the range (included) :type begin: int :param end: end position of the range (included) :type end: int """ self._begin = begin self._end = end if self._begin and self._begin < 0: raise ValueError('begin time can not be less than 0, use None for the infinity') if self._end and self._end < 0: raise ValueError('end time can not be less than 0, use None for the infinity') def __contains__(self, item): ge_begin, le_end = True, True if self._begin is not None: ge_begin = item >= self._begin if self._end is not None: le_end = item <= self._end return ge_begin and le_end class ScanPoliciesManager: """ Policy object used when scanning folders, used to decide which files to include in the list of files. Code that scans through files should at least use should_exclude_file() to decide whether each file should be included; it will check include/exclude patterns for file names, as well as patterns for excluding directories. Code that scans may optionally use should_exclude_directory() to test whether it can skip a directory completely and not bother listing the files and sub-directories in it. """ def __init__( self, exclude_dir_regexes: Iterable[str | re.Pattern] = tuple(), exclude_file_regexes: Iterable[str | re.Pattern] = tuple(), include_file_regexes: Iterable[str | re.Pattern] = tuple(), exclude_all_symlinks: bool = False, exclude_modified_before: int | None = None, exclude_modified_after: int | None = None, exclude_uploaded_before: int | None = None, exclude_uploaded_after: int | None = None, ): """ :param exclude_dir_regexes: regexes to exclude directories :param exclude_file_regexes: regexes to exclude files :param include_file_regexes: regexes to include files :param exclude_all_symlinks: if True, exclude all symlinks :param exclude_modified_before: optionally exclude file versions (both local and b2) modified before (in millis) :param exclude_modified_after: optionally exclude file versions (both local and b2) modified after (in millis) :param exclude_uploaded_before: optionally exclude b2 file versions uploaded before (in millis) :param exclude_uploaded_after: optionally exclude b2 file versions uploaded after (in millis) The regex matching priority for a given path is: 1) the path is always excluded if it's dir matches `exclude_dir_regexes`, if not then 2) the path is always included if it matches `include_file_regexes`, if not then 3) the path is excluded if it matches `exclude_file_regexes`, if not then 4) the path is included """ if include_file_regexes and not exclude_file_regexes: raise InvalidArgument( 'include_file_regexes', 'cannot be used without exclude_file_regexes at the same time', ) with check_invalid_argument( 'exclude_dir_regexes', 'wrong regex was given for excluding directories', re.error ): self._exclude_dir_set = RegexSet(exclude_dir_regexes) self._exclude_file_because_of_dir_set = RegexSet( map(convert_dir_regex_to_dir_prefix_regex, exclude_dir_regexes) ) with check_invalid_argument( 'exclude_file_regexes', 'wrong regex was given for excluding files', re.error ): self._exclude_file_set = RegexSet(exclude_file_regexes) with check_invalid_argument( 'include_file_regexes', 'wrong regex was given for including files', re.error ): self._include_file_set = RegexSet(include_file_regexes) self.exclude_all_symlinks = exclude_all_symlinks with check_invalid_argument( 'exclude_modified_before,exclude_modified_after', '', ValueError ): self._include_mod_time_range = IntegerRange( exclude_modified_before, exclude_modified_after ) with check_invalid_argument( 'exclude_uploaded_before,exclude_uploaded_after', '', ValueError ): self._include_upload_time_range = IntegerRange( exclude_uploaded_before, exclude_uploaded_after ) def should_exclude_relative_path(self, relative_path: str): if self._include_file_set.matches(relative_path): return False return self._exclude_file_set.matches(relative_path) def should_exclude_local_path(self, local_path: LocalPath): """ Whether a local path should be excluded from the scan or not. This method assumes that the directory holding the `path_` has already been checked for exclusion. """ if local_path.mod_time not in self._include_mod_time_range: return True return self.should_exclude_relative_path(local_path.relative_path) def should_exclude_b2_file_version(self, file_version: FileVersion, relative_path: str): """ Whether a b2 file version should be excluded from the scan or not. This method assumes that the directory holding the `path_` has already been checked for exclusion. """ if file_version.upload_timestamp not in self._include_upload_time_range: return True if file_version.mod_time_millis not in self._include_mod_time_range: return True return self.should_exclude_relative_path(relative_path) def should_exclude_b2_directory(self, dir_path: str): """ Given the path of a directory, relative to the scan point, decide if all of the files in it should be excluded from the scan. """ return self._exclude_dir_set.matches(dir_path) def should_exclude_local_directory(self, dir_path: str): """ Given the path of a directory, relative to the scan point, decide if all of the files in it should be excluded from the scan. """ return self._exclude_dir_set.matches(dir_path) DEFAULT_SCAN_MANAGER = ScanPoliciesManager() b2-sdk-python-2.8.0/b2sdk/_internal/scan/report.py000066400000000000000000000162401474454370000217260ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/scan/report.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging import re import threading import time from dataclasses import dataclass from io import TextIOWrapper from ..utils import format_and_scale_number from ..utils.escape import escape_control_chars logger = logging.getLogger(__name__) _REMOVE_EXTENDED_PATH_PREFIX = re.compile(r'\\\\\?\\') def _safe_path_print(path: str) -> str: """ Print a path, escaping control characters if necessary. Windows extended path prefix is removed from the path before printing for better readability. Since Windows 10 the prefix is not needed. :param path: a path to print :return: a path that can be printed """ return escape_control_chars(_REMOVE_EXTENDED_PATH_PREFIX.sub('', path)) @dataclass class ProgressReport: """ Handle reporting progress. This class is THREAD SAFE, so it can be used from parallel scan threads. """ # Minimum time between displayed updates UPDATE_INTERVAL = 0.1 stdout: TextIOWrapper # standard output file object no_progress: bool # if True, do not show progress def __post_init__(self): self.start_time = time.time() self.count = 0 self.total_done = False self.total_count = 0 self.closed = False self.lock = threading.Lock() self.current_line = '' self.encoding_warning_was_already_printed = False self._last_update_time = 0 self._update_progress() self.warnings = [] self.errors_encountered = False def close(self): """ Perform a clean-up. """ with self.lock: if not self.no_progress: self._print_line('', False) self.closed = True for warning in self.warnings: self._print_line(warning, True) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def has_errors_or_warnings(self) -> bool: """ Check if there are any errors or warnings. :return: True if there are any errors or warnings """ return self.errors_encountered or bool(self.warnings) def error(self, message: str) -> None: """ Print an error, gracefully interleaving it with a progress bar. :param message: an error message """ self.print_completion(message) self.errors_encountered = True def print_completion(self, message: str) -> None: """ Remove the progress bar, prints a message, and puts the progress bar back. :param message: an error message """ with self.lock: self._print_line(message, True) self._last_update_time = 0 self._update_progress() def update_count(self, delta: int) -> None: """ Report that items have been processed. """ with self.lock: self.count += delta self._update_progress() def _update_progress(self): if self.closed or self.no_progress: return now = time.time() interval = now - self._last_update_time if interval < self.UPDATE_INTERVAL: return self._last_update_time = now time_delta = time.time() - self.start_time rate = 0 if time_delta == 0 else int(self.count / time_delta) message = ' count: %d/%d %s' % ( self.count, self.total_count, format_and_scale_number(rate, '/s'), ) self._print_line(message, False) def _print_line(self, line: str, newline: bool) -> None: """ Print a line to stdout. :param line: a string without a \r or \n in it. :param newline: True if the output should move to a new line after this one. """ if len(line) < len(self.current_line): line += ' ' * (len(self.current_line) - len(line)) try: self.stdout.write(line) except UnicodeEncodeError as encode_error: if not self.encoding_warning_was_already_printed: self.encoding_warning_was_already_printed = True self.stdout.write( f'!WARNING! this terminal cannot properly handle progress reporting. encoding is {self.stdout.encoding}.\n' ) self.stdout.write(line.encode('ascii', 'backslashreplace').decode()) logger.warning( f'could not output the following line with encoding {self.stdout.encoding} on stdout due to {encode_error}: {line}' ) if newline: self.stdout.write('\n') self.current_line = '' else: self.stdout.write('\r') self.current_line = line self.stdout.flush() def update_total(self, delta: int) -> None: """ Report that more files have been found for comparison. :param delta: number of files found since the last check """ with self.lock: self.total_count += delta self._update_progress() def end_total(self) -> None: """ Total files count is done. Can proceed to step 2. """ with self.lock: self.total_done = True self._update_progress() def local_access_error(self, path: str) -> None: """ Add a file access error message to the list of warnings. :param path: file path """ self.warnings.append( f'WARNING: {_safe_path_print(path)} could not be accessed (broken symlink?)' ) def local_permission_error(self, path: str) -> None: """ Add a permission error message to the list of warnings. :param path: file path """ self.warnings.append( f'WARNING: {_safe_path_print(path)} could not be accessed (no permissions to read?)' ) def symlink_skipped(self, path: str) -> None: pass def circular_symlink_skipped(self, path: str) -> None: """ Add a circular symlink error message to the list of warnings. :param path: file path """ self.warnings.append( f'WARNING: {_safe_path_print(path)} is a circular symlink, which was already visited. Skipping.' ) def invalid_name(self, path: str, error: str) -> None: """ Add an invalid filename error message to the list of warnings. :param path: file path """ self.warnings.append( f'WARNING: {_safe_path_print(path)} path contains invalid name ({error}). Skipping.' ) def sample_report_run(): """ Generate a sample report. """ import sys report = ProgressReport(sys.stdout, False) for i in range(20): report.update_total(1) time.sleep(0.2) if i % 2 == 0: report.update_count(1) report.end_total() report.close() b2-sdk-python-2.8.0/b2sdk/_internal/scan/scan.py000066400000000000000000000075411474454370000213430ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/scan/scan.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from abc import ABCMeta, abstractclassmethod, abstractmethod from collections import Counter from dataclasses import dataclass, field from typing import ClassVar from ..file_version import FileVersion from .folder import AbstractFolder from .path import AbstractPath from .policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from .report import ProgressReport def zip_folders( folder_a: AbstractFolder, folder_b: AbstractFolder, reporter: ProgressReport, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER, ) -> tuple[AbstractPath | None, AbstractPath | None]: """ Iterate over all of the files in the union of two folders, matching file names. Each item is a pair (file_a, file_b) with the corresponding file in both folders. Either file (but not both) will be None if the file is in only one folder. :param b2sdk._internal.scan.folder.AbstractFolder folder_a: first folder object. :param b2sdk._internal.scan.folder.AbstractFolder folder_b: second folder object. :param reporter: reporter object :param policies_manager: policies manager object :return: yields two element tuples """ iter_a = folder_a.all_files(reporter, policies_manager) iter_b = folder_b.all_files(reporter) current_a = next(iter_a, None) current_b = next(iter_b, None) while current_a is not None or current_b is not None: if current_a is None: yield (None, current_b) current_b = next(iter_b, None) elif current_b is None: yield (current_a, None) current_a = next(iter_a, None) elif current_a.relative_path < current_b.relative_path: yield (current_a, None) current_a = next(iter_a, None) elif current_b.relative_path < current_a.relative_path: yield (None, current_b) current_b = next(iter_b, None) else: assert current_a.relative_path == current_b.relative_path yield (current_a, current_b) current_a = next(iter_a, None) current_b = next(iter_b, None) reporter.close() @dataclass(frozen=True) class AbstractScanResult(metaclass=ABCMeta): """ Some attributes of files which are meaningful for monitoring and troubleshooting. """ @abstractclassmethod def from_files(cls, *files: AbstractPath | None) -> AbstractScanResult: pass @dataclass class AbstractScanReport(metaclass=ABCMeta): """ Aggregation of valuable information about files after scanning. """ SCAN_RESULT_CLASS: ClassVar[type] = AbstractScanResult @abstractmethod def add(self, *files: AbstractPath | None) -> None: pass @dataclass class CountAndSampleScanReport(AbstractScanReport): """ Scan report which groups and counts files by their `AbstractScanResult` and also stores first and last seen examples of such files. """ counter_by_status: Counter = field(default_factory=Counter) samples_by_status_first: dict[AbstractScanResult, tuple[FileVersion, ...]] = field( default_factory=dict ) samples_by_status_last: dict[AbstractScanResult, tuple[FileVersion, ...]] = field( default_factory=dict ) def add(self, *files: AbstractPath | None) -> None: status = self.SCAN_RESULT_CLASS.from_files(*files) self.counter_by_status[status] += 1 sample = tuple(file and file.selected_version for file in files) self.samples_by_status_first.setdefault(status, sample) self.samples_by_status_last[status] = sample b2-sdk-python-2.8.0/b2sdk/_internal/session.py000066400000000000000000000531571474454370000211620ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/session.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from enum import Enum, unique from functools import partial from typing import Any from b2sdk._internal.account_info.abstract import AbstractAccountInfo from b2sdk._internal.account_info.exception import MissingAccountData from b2sdk._internal.account_info.sqlite_account_info import SqliteAccountInfo from b2sdk._internal.api_config import DEFAULT_HTTP_API_CONFIG, B2HttpApiConfig from b2sdk._internal.b2http import B2Http from b2sdk._internal.cache import AbstractCache, AuthInfoCache, DummyCache from b2sdk._internal.encryption.setting import EncryptionSetting from b2sdk._internal.exception import InvalidAuthToken, Unauthorized from b2sdk._internal.file_lock import BucketRetentionSetting, FileRetentionSetting, LegalHold from b2sdk._internal.raw_api import ALL_CAPABILITIES, REALM_URLS, LifecycleRule from b2sdk._internal.replication.setting import ReplicationConfiguration logger = logging.getLogger(__name__) @unique class TokenType(Enum): API = 'api' API_TOKEN_ONLY = 'api_token_only' UPLOAD_PART = 'upload_part' UPLOAD_SMALL = 'upload_small' class B2Session: """ A facade that supplies the correct api_url and account_auth_token to methods of underlying raw_api and reauthorizes if necessary. """ SQLITE_ACCOUNT_INFO_CLASS = staticmethod(SqliteAccountInfo) B2HTTP_CLASS = staticmethod(B2Http) def __init__( self, account_info: AbstractAccountInfo | None = None, cache: AbstractCache | None = None, api_config: B2HttpApiConfig = DEFAULT_HTTP_API_CONFIG, ): """ Initialize Session using given account info. :param account_info: an instance of :class:`~b2sdk.v2.UrlPoolAccountInfo`, or any custom class derived from :class:`~b2sdk.v2.AbstractAccountInfo` To learn more about Account Info objects, see here :class:`~b2sdk.v2.SqliteAccountInfo` :param cache: an instance of the one of the following classes: :class:`~b2sdk._internal.cache.DummyCache`, :class:`~b2sdk._internal.cache.InMemoryCache`, :class:`~b2sdk._internal.cache.AuthInfoCache`, or any custom class derived from :class:`~b2sdk._internal.cache.AbstractCache` It is used by B2Api to cache the mapping between bucket name and bucket ids. default is :class:`~b2sdk._internal.cache.DummyCache` :param api_config """ self.raw_api = api_config.raw_api_class(self.B2HTTP_CLASS(api_config)) if account_info is None: account_info = self.SQLITE_ACCOUNT_INFO_CLASS() if cache is None: if account_info: cache = AuthInfoCache(account_info) else: cache = DummyCache() self.account_info = account_info self.cache = cache self._token_callbacks = { TokenType.API: self._api_token_callback, TokenType.API_TOKEN_ONLY: self._api_token_only_callback, TokenType.UPLOAD_SMALL: self._upload_small, TokenType.UPLOAD_PART: self._upload_part, } def authorize_automatically(self): """ Perform automatic account authorization, retrieving all account data from account info object passed during initialization. """ try: self.authorize_account( self.account_info.get_realm(), self.account_info.get_application_key_id(), self.account_info.get_application_key(), ) except MissingAccountData: return False return True def authorize_account(self, realm, application_key_id, application_key): """ Perform account authorization. :param str realm: a realm to authorize account in (usually just "production") :param str application_key_id: :term:`application key ID` :param str application_key: user's :term:`application key` """ # Authorize realm_url = REALM_URLS.get(realm, realm) response = self.raw_api.authorize_account(realm_url, application_key_id, application_key) account_id = response['accountId'] storage_api_info = response['apiInfo']['storageApi'] # `allowed` object has been deprecated in the v3 of the API, but we still # construct it artificially to avoid changes in all the reliant parts. allowed = { 'bucketId': storage_api_info['bucketId'], 'bucketName': storage_api_info['bucketName'], 'capabilities': storage_api_info['capabilities'], 'namePrefix': storage_api_info['namePrefix'], } # Clear the cache if new account has been used if not self.account_info.is_same_account(account_id, realm): self.cache.clear() # Store the auth data self.account_info.set_auth_data( account_id=account_id, auth_token=response['authorizationToken'], api_url=storage_api_info['apiUrl'], download_url=storage_api_info['downloadUrl'], absolute_minimum_part_size=storage_api_info['absoluteMinimumPartSize'], recommended_part_size=storage_api_info['recommendedPartSize'], application_key=application_key, realm=realm, s3_api_url=storage_api_info['s3ApiUrl'], allowed=allowed, application_key_id=application_key_id, ) def cancel_large_file(self, file_id): return self._wrap_default_token(self.raw_api.cancel_large_file, file_id) def create_bucket( self, account_id, bucket_name, bucket_type, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, default_server_side_encryption=None, is_file_lock_enabled: bool | None = None, replication: ReplicationConfiguration | None = None, ): return self._wrap_default_token( self.raw_api.create_bucket, account_id, bucket_name, bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, default_server_side_encryption=default_server_side_encryption, is_file_lock_enabled=is_file_lock_enabled, replication=replication, ) def create_key( self, account_id, capabilities, key_name, valid_duration_seconds, bucket_id, name_prefix ): return self._wrap_default_token( self.raw_api.create_key, account_id, capabilities, key_name, valid_duration_seconds, bucket_id, name_prefix, ) def delete_key(self, application_key_id): return self._wrap_default_token(self.raw_api.delete_key, application_key_id) def delete_bucket(self, account_id, bucket_id): return self._wrap_default_token(self.raw_api.delete_bucket, account_id, bucket_id) def delete_file_version(self, file_id, file_name, bypass_governance: bool = False): return self._wrap_default_token( self.raw_api.delete_file_version, file_id, file_name, bypass_governance ) def download_file_from_url( self, url: str, range_: tuple[int, int] | None = None, encryption: EncryptionSetting | None = None, ): return self._wrap_token( self.raw_api.download_file_from_url, TokenType.API_TOKEN_ONLY, url, range_=range_, encryption=encryption, ) def finish_large_file(self, file_id, part_sha1_array): return self._wrap_default_token(self.raw_api.finish_large_file, file_id, part_sha1_array) def get_download_authorization(self, bucket_id, file_name_prefix, valid_duration_in_seconds): return self._wrap_default_token( self.raw_api.get_download_authorization, bucket_id, file_name_prefix, valid_duration_in_seconds, ) def get_file_info_by_id(self, file_id: str) -> dict[str, Any]: return self._wrap_default_token(self.raw_api.get_file_info_by_id, file_id) def get_file_info_by_name(self, bucket_name: str, file_name: str) -> dict[str, Any]: return self._wrap_default_token(self.raw_api.get_file_info_by_name, bucket_name, file_name) def get_upload_url(self, bucket_id): return self._wrap_default_token(self.raw_api.get_upload_url, bucket_id) def get_upload_part_url(self, file_id): return self._wrap_default_token(self.raw_api.get_upload_part_url, file_id) def hide_file(self, bucket_id, file_name): return self._wrap_default_token(self.raw_api.hide_file, bucket_id, file_name) def list_buckets(self, account_id, bucket_id=None, bucket_name=None): return self._wrap_default_token( self.raw_api.list_buckets, account_id, bucket_id=bucket_id, bucket_name=bucket_name, ) def list_file_names( self, bucket_id, start_file_name=None, max_file_count=None, prefix=None, ): return self._wrap_default_token( self.raw_api.list_file_names, bucket_id, start_file_name=start_file_name, max_file_count=max_file_count, prefix=prefix, ) def list_file_versions( self, bucket_id, start_file_name=None, start_file_id=None, max_file_count=None, prefix=None, ): return self._wrap_default_token( self.raw_api.list_file_versions, bucket_id, start_file_name=start_file_name, start_file_id=start_file_id, max_file_count=max_file_count, prefix=prefix, ) def list_keys(self, account_id, max_key_count=None, start_application_key_id=None): return self._wrap_default_token( self.raw_api.list_keys, account_id, max_key_count=max_key_count, start_application_key_id=start_application_key_id, ) def list_parts(self, file_id, start_part_number, max_part_count): return self._wrap_default_token( self.raw_api.list_parts, file_id, start_part_number, max_part_count ) def list_unfinished_large_files( self, bucket_id, start_file_id=None, max_file_count=None, prefix=None, ): return self._wrap_default_token( self.raw_api.list_unfinished_large_files, bucket_id, start_file_id=start_file_id, max_file_count=max_file_count, prefix=prefix, ) def start_large_file( self, bucket_id, file_name, content_type, file_info, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): return self._wrap_default_token( self.raw_api.start_large_file, bucket_id, file_name, content_type, file_info, server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) def update_bucket( self, account_id, bucket_id, bucket_type=None, bucket_info=None, cors_rules=None, lifecycle_rules: list[LifecycleRule] | None = None, if_revision_is=None, default_server_side_encryption: EncryptionSetting | None = None, default_retention: BucketRetentionSetting | None = None, replication: ReplicationConfiguration | None = None, is_file_lock_enabled: bool | None = None, ): return self._wrap_default_token( self.raw_api.update_bucket, account_id, bucket_id, bucket_type=bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, replication=replication, is_file_lock_enabled=is_file_lock_enabled, ) def upload_file( self, bucket_id, file_name, content_length, content_type, content_sha1, file_info, data_stream, server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): return self._wrap_token( self.raw_api.upload_file, TokenType.UPLOAD_SMALL, bucket_id, file_name, content_length, content_type, content_sha1, file_info, data_stream, server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) def upload_part( self, file_id, part_number, content_length, sha1_sum, input_stream, server_side_encryption: EncryptionSetting | None = None, ): return self._wrap_token( self.raw_api.upload_part, TokenType.UPLOAD_PART, file_id, part_number, content_length, sha1_sum, input_stream, server_side_encryption, ) def get_download_url_by_id(self, file_id): return self.raw_api.get_download_url_by_id(self.account_info.get_download_url(), file_id) def get_download_url_by_name(self, bucket_name, file_name): return self.raw_api.get_download_url_by_name( self.account_info.get_download_url(), bucket_name, file_name ) def copy_file( self, source_file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_bucket_id=None, destination_server_side_encryption: EncryptionSetting | None = None, source_server_side_encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, ): return self._wrap_default_token( self.raw_api.copy_file, source_file_id, new_file_name, bytes_range=bytes_range, metadata_directive=metadata_directive, content_type=content_type, file_info=file_info, destination_bucket_id=destination_bucket_id, destination_server_side_encryption=destination_server_side_encryption, source_server_side_encryption=source_server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, ) def copy_part( self, source_file_id, large_file_id, part_number, bytes_range=None, destination_server_side_encryption: EncryptionSetting | None = None, source_server_side_encryption: EncryptionSetting | None = None, ): return self._wrap_default_token( self.raw_api.copy_part, source_file_id, large_file_id, part_number, bytes_range=bytes_range, destination_server_side_encryption=destination_server_side_encryption, source_server_side_encryption=source_server_side_encryption, ) def _wrap_default_token(self, raw_api_method, *args, **kwargs): return self._wrap_token(raw_api_method, TokenType.API, *args, **kwargs) def _wrap_token(self, raw_api_method, token_type, *args, **kwargs): callback = self._token_callbacks[token_type] partial_callback = partial(callback, raw_api_method, *args, **kwargs) return self._execute_with_auth_retry(partial_callback) def _api_token_callback(self, raw_api_method, *args, **kwargs): api_url = self.account_info.get_api_url() account_auth_token = self.account_info.get_account_auth_token() return raw_api_method(api_url, account_auth_token, *args, **kwargs) def _api_token_only_callback(self, raw_api_method, *args, **kwargs): account_auth_token = self.account_info.get_account_auth_token() return raw_api_method(account_auth_token, *args, **kwargs) def _execute_with_auth_retry(self, callback, first_attempt: bool = True): try: return callback() except InvalidAuthToken: if first_attempt and self.authorize_automatically(): return self._execute_with_auth_retry(callback, first_attempt=False) raise except Unauthorized as exc: # When performing HEAD requests, the api can return an empty response # with a 401 status code, which may or may not be related to expired # token, thus we also try to re-authorize if explicit `unauthorized` # code is missing if not exc.code and first_attempt and self.authorize_automatically(): return self._execute_with_auth_retry(callback, first_attempt=False) raise self._add_app_key_info_to_unauthorized(exc) def _add_app_key_info_to_unauthorized(self, unauthorized): """ Take an Unauthorized error and adds information from the application key about why it might have failed. """ # What's allowed? allowed = self.account_info.get_allowed() capabilities = allowed['capabilities'] bucket_name = allowed['bucketName'] name_prefix = allowed['namePrefix'] # Make a list of messages about the application key restrictions key_messages = [] if set(capabilities) != set(ALL_CAPABILITIES): key_messages.append("with capabilities '" + ','.join(capabilities) + "'") if bucket_name is not None: key_messages.append("restricted to bucket '" + bucket_name + "'") if name_prefix is not None: key_messages.append("restricted to files that start with '" + name_prefix + "'") if not key_messages: key_messages.append('with no restrictions') # Make a new message new_message = unauthorized.message or 'unauthorized' new_message += ' for application key ' + ', '.join(key_messages) return Unauthorized(new_message, unauthorized.code) def _get_upload_data(self, bucket_id): """ Take ownership of an upload URL / auth token for the bucket and return it. """ account_info = self.account_info upload_url, upload_auth_token = account_info.take_bucket_upload_url(bucket_id) if None not in (upload_url, upload_auth_token): return upload_url, upload_auth_token response = self.get_upload_url(bucket_id) return response['uploadUrl'], response['authorizationToken'] def _get_upload_part_data(self, file_id): """ Make sure that we have an upload URL and auth token for the given bucket and return it. """ account_info = self.account_info upload_url, upload_auth_token = account_info.take_large_file_upload_url(file_id) if None not in (upload_url, upload_auth_token): return upload_url, upload_auth_token response = self.get_upload_part_url(file_id) return response['uploadUrl'], response['authorizationToken'] def _upload_small(self, f, bucket_id, *args, **kwargs): upload_url, upload_auth_token = self._get_upload_data(bucket_id) response = f(upload_url, upload_auth_token, *args, **kwargs) self.account_info.put_bucket_upload_url(bucket_id, upload_url, upload_auth_token) return response def _upload_part(self, f, file_id, *args, **kwargs): upload_url, upload_auth_token = self._get_upload_part_data(file_id) response = f(upload_url, upload_auth_token, *args, **kwargs) self.account_info.put_large_file_upload_url(file_id, upload_url, upload_auth_token) return response def update_file_retention( self, file_id, file_name, file_retention: FileRetentionSetting, bypass_governance: bool = False, ): return self._wrap_default_token( self.raw_api.update_file_retention, file_id, file_name, file_retention, bypass_governance, ) def update_file_legal_hold( self, file_id, file_name, legal_hold: LegalHold, ): return self._wrap_default_token( self.raw_api.update_file_legal_hold, file_id, file_name, legal_hold, ) def get_bucket_notification_rules(self, bucket_id): return self._wrap_default_token(self.raw_api.get_bucket_notification_rules, bucket_id) def set_bucket_notification_rules(self, bucket_id, rules): return self._wrap_default_token( self.raw_api.set_bucket_notification_rules, bucket_id, rules ) b2-sdk-python-2.8.0/b2sdk/_internal/stream/000077500000000000000000000000001474454370000204055ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/stream/__init__.py000066400000000000000000000011471474454370000225210ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/stream/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from .hashing import StreamWithHash from .progress import ReadingStreamWithProgress, WritingStreamWithProgress from .range import RangeOfInputStream __all__ = [ 'RangeOfInputStream', 'ReadingStreamWithProgress', 'StreamWithHash', 'WritingStreamWithProgress', ] b2-sdk-python-2.8.0/b2sdk/_internal/stream/base.py000066400000000000000000000010231474454370000216650ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/stream/base.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import io class ReadOnlyStreamMixin: def writeable(self): return False def write(self, data): raise io.UnsupportedOperation('Cannot accept a write to a read-only stream') b2-sdk-python-2.8.0/b2sdk/_internal/stream/chained.py000066400000000000000000000122221474454370000223510ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/stream/chained.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import io from abc import ABCMeta, abstractmethod from b2sdk._internal.stream.base import ReadOnlyStreamMixin class ChainedStream(ReadOnlyStreamMixin, io.IOBase): """Chains multiple streams in single stream, sort of what :py:class:`itertools.chain` does for iterators. Cleans up buffers of underlying streams when closed. Can be seeked to beginning (when retrying upload, for example). Closes underlying streams as soon as they reaches EOF, but clears their buffers when the chained stream is closed for underlying streams that follow :py:class:`b2sdk.v2.StreamOpener` cleanup interface, for example :py:class:`b2sdk.v2.CachedBytesStreamOpener` """ def __init__(self, stream_openers): """ :param list stream_openeres: list of callables that return opened streams """ stream_openers = list(stream_openers) if not stream_openers: raise ValueError('chain_links cannot be empty') self.stream_openers = stream_openers self._stream_openers_iterator = iter(self.stream_openers) self._current_stream = None self._pos = 0 super().__init__() @property def stream(self): """Return currently processed stream.""" if self._current_stream is None: self._next_stream() return self._current_stream def _reset_chain(self): if self._current_stream is not None: self._current_stream.close() self._current_stream = None self._stream_openers_iterator = iter(self.stream_openers) self._pos = 0 def _next_stream(self): next_stream_opener = next(self._stream_openers_iterator, None) if next_stream_opener is not None: if self._current_stream is not None: self._current_stream.close() self._current_stream = next_stream_opener() def seekable(self): return True def tell(self): return self._pos def seek(self, pos, whence=0): """ Resets stream to the beginning. :param int pos: only allowed value is ``0`` :param int whence: only allowed value is ``0`` """ if pos != 0 or whence != 0: raise io.UnsupportedOperation('Chained stream can only be seeked to beginning') self._reset_chain() return self.tell() def readable(self): return True def read(self, size=None): """ Read at most `size` bytes from underlying streams, or all available data, if `size` is None or negative. Open the streams only when their data is needed, and possibly leave them open and part-way read for further reading - by subsequent calls to this method. :param int,None size: number of bytes to read. If omitted, ``None``, or negative data is read and returned until EOF from final stream is reached :return: data read from the stream """ byte_arrays = [] if size < 0 or size is None: while 1: current_stream = self.stream buff = current_stream.read() byte_arrays.append(buff) if not buff: self._next_stream() if self.stream is current_stream: break else: remaining = size while 1: current_stream = self.stream buff = current_stream.read(remaining) byte_arrays.append(buff) remaining -= len(buff) if remaining == 0: # no need to open any other streams - we're satisfied break if not buff: self._next_stream() if self.stream is current_stream: break if not byte_arrays: data = byte_arrays[0] else: data = b''.join(byte_arrays) self._pos += len(data) return data def close(self): if self._current_stream is not None: self._current_stream.close() for stream_opener in self.stream_openers: if hasattr(stream_opener, 'cleanup'): stream_opener.cleanup() super().close() class StreamOpener(metaclass=ABCMeta): """Abstract class to define stream opener with cleanup.""" @abstractmethod def __call__(self): """Create or open the stream to read and return. Can be called multiple times, but streamed data may be cached and reused. """ def cleanup(self): """Clean up stream opener after chained stream closes. Can be used for cleaning cached data that are stored in memory to allow resetting chained stream without getting this data more than once, eg. data downloaded from external source. """ b2-sdk-python-2.8.0/b2sdk/_internal/stream/hashing.py000066400000000000000000000045271474454370000224100ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/stream/hashing.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import hashlib import io from b2sdk._internal.stream.base import ReadOnlyStreamMixin from b2sdk._internal.stream.wrapper import StreamWithLengthWrapper class StreamWithHash(ReadOnlyStreamMixin, StreamWithLengthWrapper): """ Wrap a file-like object, calculates SHA1 while reading and appends hash at the end. """ def __init__(self, stream, stream_length=None): """ :param stream: the stream to read from """ self.digest = self.get_digest() total_length = None if stream_length is not None: total_length = stream_length + self.digest.digest_size * 2 super().__init__(stream, length=total_length) self.hash = None self.hash_read = 0 def seek(self, pos, whence=0): """ Seek to a given position in the stream. :param int pos: position in the stream """ if pos != 0 or whence != 0: raise io.UnsupportedOperation('Stream with hash can only be seeked to beginning') self.digest = self.get_digest() self.hash = None self.hash_read = 0 return super().seek(0) def read(self, size=None): """ Read data from the stream. :param int size: number of bytes to read :return: read data :rtype: bytes|None """ data = b'' if self.hash is None: data = super().read(size) # Update hash self.digest.update(data) # Check for end of stream if size is None or len(data) < size: self.hash = self.digest.hexdigest() if size is not None: size -= len(data) if self.hash is not None: # The end of stream was reached, return hash now size = size or len(self.hash) data += str.encode(self.hash[self.hash_read : self.hash_read + size]) self.hash_read += size return data @classmethod def get_digest(cls): return hashlib.sha1() b2-sdk-python-2.8.0/b2sdk/_internal/stream/progress.py000066400000000000000000000060341474454370000226260ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/stream/progress.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal.stream.wrapper import StreamWrapper class AbstractStreamWithProgress(StreamWrapper): """ Wrap a file-like object and updates a ProgressListener as data is read / written. In the abstract class, read and write methods do not update the progress - child classes shall do it. """ def __init__(self, stream, progress_listener, offset=0): """ :param stream: the stream to read from or write to :param b2sdk.v2.AbstractProgressListener progress_listener: the listener that we tell about progress :param int offset: the starting byte offset in the file """ super().__init__(stream) assert progress_listener is not None self.progress_listener = progress_listener self.bytes_completed = 0 self.offset = offset def _progress_update(self, delta): self.bytes_completed += delta self.progress_listener.bytes_completed(self.bytes_completed + self.offset) def __str__(self): return str(self.stream) class ReadingStreamWithProgress(AbstractStreamWithProgress): """ Wrap a file-like object, updates progress while reading. """ def __init__(self, *args, **kwargs): length = kwargs.pop('length', None) super().__init__(*args, **kwargs) self.length = length def read(self, size=None): """ Read data from the stream. :param int size: number of bytes to read :return: data read from the stream """ data = super().read(size) self._progress_update(len(data)) return data def seek(self, pos, whence=0): pos = super().seek(pos, whence=whence) # reset progress to current stream position - assumption is that ReadingStreamWithProgress would not be used # for random access streams, and seek is only used to reset stream to beginning to retry file upload # and multipart file upload would open and use different file descriptor for each part; # this logic cannot be used for WritingStreamWithProgress because multipart download has to use # single file descriptor and synchronize writes so seeking cannot be understood there as progress reset # and writing progress is always monotonic self.bytes_completed = pos return pos def __len__(self): return self.length class WritingStreamWithProgress(AbstractStreamWithProgress): """ Wrap a file-like object; updates progress while writing. """ def write(self, data): """ Write data to the stream. :param bytes data: data to write to the stream """ self._progress_update(len(data)) return super().write(data) b2-sdk-python-2.8.0/b2sdk/_internal/stream/range.py000066400000000000000000000050451474454370000220570ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/stream/range.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import io from b2sdk._internal.stream.base import ReadOnlyStreamMixin from b2sdk._internal.stream.wrapper import StreamWithLengthWrapper class RangeOfInputStream(ReadOnlyStreamMixin, StreamWithLengthWrapper): """ Wrap a file-like object (read only) and read the selected range of the file. """ def __init__(self, stream, offset, length): """ :param stream: a seekable stream :param int offset: offset in the stream :param int length: max number of bytes to read """ super().__init__(stream, length) self.offset = offset self.relative_pos = 0 self.stream.seek(self.offset) def seek(self, pos, whence=0): """ Seek to a given position in the stream. :param int pos: position in the stream relative to steam offset :return: new position relative to stream offset :rtype: int """ if whence != 0: raise io.UnsupportedOperation('only SEEK_SET is supported') abs_pos = super().seek(self.offset + pos) self.relative_pos = abs_pos - self.offset return self.tell() def tell(self): """ Return current stream position relative to offset. :rtype: int """ return self.relative_pos def read(self, size=None): """ Read data from the stream. :param int size: number of bytes to read :return: data read from the stream :rtype: bytes """ remaining = max(0, self.length - self.relative_pos) if size is None: to_read = remaining else: to_read = min(size, remaining) data = self.stream.read(to_read) self.relative_pos += len(data) return data def close(self): super().close() # TODO: change the use cases of this class to close the file objects passed to it, instead of having # RangeOfInputStream close it's members upon garbage collection self.stream.close() def wrap_with_range(stream, stream_length, range_offset, range_length): if range_offset == 0 and range_length == stream_length: return stream return RangeOfInputStream(stream, range_offset, range_length) b2-sdk-python-2.8.0/b2sdk/_internal/stream/wrapper.py000066400000000000000000000044551474454370000224470ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/stream/wrapper.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import io class StreamWrapper(io.IOBase): """ Wrapper for a file-like object. """ def __init__(self, stream): """ :param stream: the stream to read from or write to """ self.stream = stream super().__init__() def seekable(self): return self.stream.seekable() def seek(self, pos, whence=0): """ Seek to a given position in the stream. :param int pos: position in the stream :return: new absolute position :rtype: int """ return self.stream.seek(pos, whence) def tell(self): """ Return current stream position. :rtype: int """ return self.stream.tell() def truncate(self, size=None): return self.stream.truncate(size) def flush(self): """ Flush the stream. """ self.stream.flush() @property def closed(self): return self.stream.closed def readable(self): return self.stream.readable() def read(self, size=None): """ Read data from the stream. :param int size: number of bytes to read :return: data read from the stream """ if size is not None: return self.stream.read(size) else: return self.stream.read() def writable(self): return self.stream.writable() def write(self, data): """ Write data to the stream. :param data: a data to write to the stream """ return self.stream.write(data) class StreamWithLengthWrapper(StreamWrapper): """ Wrapper for a file-like object that supports `__len__` interface """ def __init__(self, stream, length=None): """ :param stream: the stream to read from or write to :param int length: length of the stream """ super().__init__(stream) self.length = length def __len__(self): return self.length b2-sdk-python-2.8.0/b2sdk/_internal/sync/000077500000000000000000000000001474454370000200665ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/sync/__init__.py000066400000000000000000000005161474454370000222010ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/sync/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/sync/action.py000066400000000000000000000426651474454370000217320ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/sync/action.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import contextlib import functools import logging import os from abc import ABCMeta, abstractmethod from contextlib import suppress from ..bucket import Bucket from ..file_version import FileVersion from ..http_constants import SRC_LAST_MODIFIED_MILLIS from ..scan.path import B2Path from ..sync.report import ProgressReport, SyncReport from ..transfer.outbound.outbound_source import OutboundTransferSource from ..transfer.outbound.upload_source import UploadSourceLocalFile from ..utils.escape import escape_control_chars from .encryption_provider import AbstractSyncEncryptionSettingsProvider from .report import SyncFileReporter logger = logging.getLogger(__name__) class AbstractAction(metaclass=ABCMeta): """ An action to take, such as uploading, downloading, or deleting a file. Multi-threaded tasks create a sequence of Actions which are then run by a pool of threads. An action can depend on other actions completing. An example of this is making sure a CreateBucketAction happens before an UploadFileAction. """ def run(self, bucket: Bucket, reporter: ProgressReport, dry_run: bool = False): """ Main action routine. :param bucket: a Bucket object :type bucket: b2sdk._internal.bucket.Bucket :param reporter: a place to report errors :param dry_run: if True, perform a dry run :type dry_run: bool """ try: if not dry_run: self.do_action(bucket, reporter) self.do_report(bucket, reporter) except Exception as e: logger.exception('an exception occurred in a sync action') reporter.error(str(self) + ': ' + repr(e) + ' ' + str(e)) raise # Re-throw so we can identify failed actions @abstractmethod def get_bytes(self) -> int: """ Return the number of bytes to transfer for this action. """ @abstractmethod def do_action(self, bucket: Bucket, reporter: ProgressReport) -> None: """ Perform the action, returning only after the action is completed. :param bucket: a Bucket object :param reporter: a place to report errors """ @abstractmethod def do_report(self, bucket: Bucket, reporter: ProgressReport) -> None: """ Report the action performed. :param bucket: a Bucket object :param reporter: a place to report errors """ class B2UploadAction(AbstractAction): """ File uploading action. """ def __init__( self, local_full_path: str, relative_name: str, b2_file_name: str, mod_time_millis: int, size: int, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider, ): """ :param local_full_path: a local file path :param relative_name: a relative file name :param b2_file_name: a name of a new remote file :param mod_time_millis: file modification time in milliseconds :param size: a file size :param encryption_settings_provider: encryption setting provider """ self.local_full_path = local_full_path self.relative_name = relative_name self.b2_file_name = b2_file_name self.mod_time_millis = mod_time_millis self.size = size self.encryption_settings_provider = encryption_settings_provider self.large_file_sha1 = None def get_bytes(self) -> int: """ Return file size. """ return self.size @functools.cached_property def _upload_source(self) -> UploadSourceLocalFile: """Upload source if the file was to be uploaded in full""" # NOTE: We're caching this to ensure that sha1 is not recalculated. return UploadSourceLocalFile(self.local_full_path) def get_all_sources(self) -> list[OutboundTransferSource]: """Get list of sources required to complete this upload""" return [self._upload_source] def do_action(self, bucket: Bucket, reporter: ProgressReport) -> None: """ Perform the uploading action, returning only after the action is completed. :param bucket: a Bucket object :param reporter: a place to report errors """ if reporter: progress_listener = SyncFileReporter(reporter) else: progress_listener = None file_info = {SRC_LAST_MODIFIED_MILLIS: str(self.mod_time_millis)} encryption = self.encryption_settings_provider.get_setting_for_upload( bucket=bucket, b2_file_name=self.b2_file_name, file_info=file_info, length=self.size, ) sources = self.get_all_sources() large_file_sha1 = None if len(sources) > 1: # The upload will be incremental, calculate the large_file_sha1 large_file_sha1 = self._upload_source.get_content_sha1() with contextlib.ExitStack() as exit_stack: if progress_listener: exit_stack.enter_context(progress_listener) bucket.concatenate( sources, self.b2_file_name, progress_listener=progress_listener, file_info=file_info, encryption=encryption, large_file_sha1=large_file_sha1, ) def do_report(self, bucket: Bucket, reporter: ProgressReport) -> None: """ Report the uploading action performed. :param bucket: a Bucket object :param reporter: a place to report errors """ reporter.print_completion(f'upload {escape_control_chars(self.relative_name)}') def __str__(self) -> str: return f'b2_upload({self.local_full_path}, {self.b2_file_name}, {self.mod_time_millis})' class B2IncrementalUploadAction(B2UploadAction): def __init__( self, local_full_path: str, relative_name: str, b2_file_name: str, mod_time_millis: int, size: int, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider, file_version: FileVersion | None = None, absolute_minimum_part_size: int | None = None, ): """ :param local_full_path: a local file path :param relative_name: a relative file name :param b2_file_name: a name of a new remote file :param mod_time_millis: file modification time in milliseconds :param size: a file size :param encryption_settings_provider: encryption setting provider :param file_version: version of file currently on the server :param absolute_minimum_part_size: minimum file part size for large files """ super().__init__( local_full_path, relative_name, b2_file_name, mod_time_millis, size, encryption_settings_provider, ) self.file_version = file_version self.absolute_minimum_part_size = absolute_minimum_part_size def get_all_sources(self) -> list[OutboundTransferSource]: return self._upload_source.get_incremental_sources( self.file_version, self.absolute_minimum_part_size ) class B2HideAction(AbstractAction): def __init__(self, relative_name: str, b2_file_name: str): """ :param relative_name: a relative file name :param b2_file_name: a name of a remote file """ self.relative_name = relative_name self.b2_file_name = b2_file_name def get_bytes(self) -> int: """ Return file size. :return: always zero :rtype: int """ return 0 def do_action(self, bucket: Bucket, reporter: ProgressReport) -> None: """ Perform the hiding action, returning only after the action is completed. :param bucket: a Bucket object :param reporter: a place to report errors """ bucket.hide_file(self.b2_file_name) def do_report(self, bucket: Bucket, reporter: SyncReport): """ Report the hiding action performed. :param bucket: a Bucket object :param reporter: a place to report errors """ reporter.update_transfer(1, 0) reporter.print_completion(f'hide {escape_control_chars(self.relative_name)}') def __str__(self) -> str: return f'b2_hide({self.b2_file_name})' class B2DownloadAction(AbstractAction): def __init__( self, source_path: B2Path, b2_file_name: str, local_full_path: str, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider, ): """ :param source_path: the file to be downloaded :param b2_file_name: b2_file_name :param local_full_path: a local file path :param encryption_settings_provider: encryption setting provider """ self.source_path = source_path self.b2_file_name = b2_file_name self.local_full_path = local_full_path self.encryption_settings_provider = encryption_settings_provider def get_bytes(self) -> int: """ Return file size. """ return self.source_path.size def _ensure_directory_existence(self) -> None: # TODO: this can fail to multiple reasons (e.g. path is a file, permissions etc). # We could provide nice exceptions for it. parent_dir = os.path.dirname(self.local_full_path) if not os.path.isdir(parent_dir): with suppress(OSError): os.makedirs(parent_dir) if not os.path.isdir(parent_dir): raise Exception(f'could not create directory {parent_dir}') def do_action(self, bucket: Bucket, reporter: ProgressReport) -> None: """ Perform the downloading action, returning only after the action is completed. :param bucket: a Bucket object :param reporter: a place to report errors """ self._ensure_directory_existence() if reporter: progress_listener = SyncFileReporter(reporter) else: progress_listener = None # Download the file to a .tmp file download_path = self.local_full_path + '.b2.sync.tmp' encryption = self.encryption_settings_provider.get_setting_for_download( bucket=bucket, file_version=self.source_path.selected_version, ) with contextlib.ExitStack() as exit_stack: if progress_listener: exit_stack.enter_context(progress_listener) downloaded_file = bucket.download_file_by_id( self.source_path.selected_version.id_, progress_listener=progress_listener, encryption=encryption, ) downloaded_file.save_to(download_path) # Move the file into place with suppress(OSError): os.unlink(self.local_full_path) os.rename(download_path, self.local_full_path) def do_report(self, bucket: Bucket, reporter: ProgressReport) -> None: """ Report the downloading action performed. :param bucket: a Bucket object :param reporter: a place to report errors """ reporter.print_completion('dnload ' + self.source_path.relative_path) def __str__(self) -> str: return 'b2_download(%s, %s, %s, %d)' % ( self.b2_file_name, self.source_path.selected_version.id_, self.local_full_path, self.source_path.mod_time, ) class B2CopyAction(AbstractAction): """ File copying action. """ def __init__( self, b2_file_name: str, source_path: B2Path, dest_b2_file_name, source_bucket: Bucket, destination_bucket: Bucket, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider, ): """ :param b2_file_name: a b2_file_name :param source_path: the file to be copied :param dest_b2_file_name: a name of a destination remote file :param source_bucket: bucket to copy from :param destination_bucket: bucket to copy to :param encryption_settings_provider: encryption setting provider """ self.b2_file_name = b2_file_name self.source_path = source_path self.dest_b2_file_name = dest_b2_file_name self.encryption_settings_provider = encryption_settings_provider self.source_bucket = source_bucket self.destination_bucket = destination_bucket def get_bytes(self) -> int: """ Return file size. """ return self.source_path.size def do_action(self, bucket: Bucket, reporter: ProgressReport) -> None: """ Perform the copying action, returning only after the action is completed. :param bucket: a Bucket object :param reporter: a place to report errors """ if reporter: progress_listener = SyncFileReporter(reporter) else: progress_listener = None source_encryption = self.encryption_settings_provider.get_source_setting_for_copy( bucket=self.source_bucket, source_file_version=self.source_path.selected_version, ) destination_encryption = self.encryption_settings_provider.get_destination_setting_for_copy( bucket=self.destination_bucket, source_file_version=self.source_path.selected_version, dest_b2_file_name=self.dest_b2_file_name, ) with contextlib.ExitStack() as exit_stack: if progress_listener: exit_stack.enter_context(progress_listener) bucket.copy( self.source_path.selected_version.id_, self.dest_b2_file_name, length=self.source_path.size, progress_listener=progress_listener, destination_encryption=destination_encryption, source_encryption=source_encryption, source_file_info=self.source_path.selected_version.file_info, source_content_type=self.source_path.selected_version.content_type, ) def do_report(self, bucket: Bucket, reporter: ProgressReport) -> None: """ Report the copying action performed. :param bucket: a Bucket object :param reporter: a place to report errors """ reporter.print_completion('copy ' + self.source_path.relative_path) def __str__(self) -> str: return 'b2_copy(%s, %s, %s, %d)' % ( self.b2_file_name, self.source_path.selected_version.id_, self.dest_b2_file_name, self.source_path.mod_time, ) class B2DeleteAction(AbstractAction): def __init__(self, relative_name: str, b2_file_name: str, file_id: str, note: str): """ :param relative_name: a relative file name :param b2_file_name: a name of a remote file :param file_id: a file ID :param note: a deletion note """ self.relative_name = relative_name self.b2_file_name = b2_file_name self.file_id = file_id self.note = note def get_bytes(self) -> int: """ Return file size. :return: always zero """ return 0 def do_action(self, bucket: Bucket, reporter: ProgressReport): """ Perform the deleting action, returning only after the action is completed. :param bucket: a Bucket object :param reporter: a place to report errors """ bucket.api.delete_file_version(self.file_id, self.b2_file_name) def do_report(self, bucket: Bucket, reporter: SyncReport): """ Report the deleting action performed. :param bucket: a Bucket object :param reporter: a place to report errors """ reporter.update_transfer(1, 0) reporter.print_completion(f'delete {escape_control_chars(self.relative_name)} {self.note}') def __str__(self) -> str: return f'b2_delete({self.b2_file_name}, {self.file_id}, {self.note})' class LocalDeleteAction(AbstractAction): def __init__(self, relative_name: str, full_path: str): """ :param relative_name: a relative file name :param full_path: a full local path """ self.relative_name = relative_name self.full_path = full_path def get_bytes(self) -> int: """ Return file size. :return: always zero """ return 0 def do_action(self, bucket: Bucket, reporter: ProgressReport): """ Perform the deleting of a local file action, returning only after the action is completed. :param bucket: a Bucket object :param reporter: a place to report errors """ os.unlink(self.full_path) def do_report(self, bucket: Bucket, reporter: SyncReport): """ Report the deleting of a local file action performed. :param bucket: a Bucket object :param reporter: a place to report errors """ reporter.update_transfer(1, 0) reporter.print_completion(f'delete {escape_control_chars(self.relative_name)}') def __str__(self) -> str: return f'local_delete({self.full_path})' b2-sdk-python-2.8.0/b2sdk/_internal/sync/encryption_provider.py000066400000000000000000000074751474454370000245610ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/sync/encryption_provider.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from abc import ABCMeta, abstractmethod from ..bucket import Bucket from ..encryption.setting import EncryptionSetting from ..file_version import FileVersion class AbstractSyncEncryptionSettingsProvider(metaclass=ABCMeta): """ Object which provides an appropriate EncryptionSetting object for sync, i.e. complex operations with multiple sources and destinations """ @abstractmethod def get_setting_for_upload( self, bucket: Bucket, b2_file_name: str, file_info: dict | None, length: int, ) -> EncryptionSetting | None: """ Return an EncryptionSetting for uploading an object or None if server should decide. """ @abstractmethod def get_source_setting_for_copy( self, bucket: Bucket, source_file_version: FileVersion, ) -> EncryptionSetting | None: """ Return an EncryptionSetting for a source of copying an object or None if not required """ @abstractmethod def get_destination_setting_for_copy( self, bucket: Bucket, dest_b2_file_name: str, source_file_version: FileVersion, target_file_info: dict | None = None, ) -> EncryptionSetting | None: """ Return an EncryptionSetting for a destination for copying an object or None if server should decide """ @abstractmethod def get_setting_for_download( self, bucket: Bucket, file_version: FileVersion, ) -> EncryptionSetting | None: """ Return an EncryptionSetting for downloading an object from, or None if not required """ class ServerDefaultSyncEncryptionSettingsProvider(AbstractSyncEncryptionSettingsProvider): """ Encryption settings provider which assumes setting-less reads and a bucket default for writes. """ def get_setting_for_upload(self, *args, **kwargs) -> None: return None def get_source_setting_for_copy(self, *args, **kwargs) -> None: return None def get_destination_setting_for_copy(self, *args, **kwargs) -> None: return None def get_setting_for_download(self, *args, **kwargs) -> None: return None SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER = ServerDefaultSyncEncryptionSettingsProvider() class BasicSyncEncryptionSettingsProvider(AbstractSyncEncryptionSettingsProvider): """ Basic encryption setting provider that supports exactly one encryption setting per bucket for reading and one encryption setting per bucket for writing """ def __init__( self, read_bucket_settings: dict[str, EncryptionSetting | None], write_bucket_settings: dict[str, EncryptionSetting | None], ): self.read_bucket_settings = read_bucket_settings self.write_bucket_settings = write_bucket_settings def get_setting_for_upload(self, bucket, *args, **kwargs) -> EncryptionSetting | None: return self.write_bucket_settings.get(bucket.name) def get_source_setting_for_copy(self, bucket, *args, **kwargs) -> None: return self.read_bucket_settings.get(bucket.name) def get_destination_setting_for_copy(self, bucket, *args, **kwargs) -> EncryptionSetting | None: return self.write_bucket_settings.get(bucket.name) def get_setting_for_download(self, bucket, *args, **kwargs) -> None: return self.read_bucket_settings.get(bucket.name) def __repr__(self): return f'<{self.__class__.__name__}:{self.bucket_settings}>' b2-sdk-python-2.8.0/b2sdk/_internal/sync/exception.py000066400000000000000000000006461474454370000224440ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/sync/exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from ..exception import B2SimpleError class IncompleteSync(B2SimpleError): pass b2-sdk-python-2.8.0/b2sdk/_internal/sync/policy.py000066400000000000000000000436671474454370000217570ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/sync/policy.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from abc import ABCMeta, abstractmethod from enum import Enum, unique from typing import cast from ..exception import DestFileNewer from ..scan.exception import InvalidArgument from ..scan.folder import AbstractFolder, B2Folder from ..scan.path import AbstractPath, B2Path from ..transfer.outbound.upload_source import UploadMode from .action import ( B2CopyAction, B2DeleteAction, B2DownloadAction, B2HideAction, B2IncrementalUploadAction, B2UploadAction, LocalDeleteAction, ) from .encryption_provider import ( SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, AbstractSyncEncryptionSettingsProvider, ) ONE_DAY_IN_MS = 24 * 60 * 60 * 1000 logger = logging.getLogger(__name__) @unique class NewerFileSyncMode(Enum): """Mode of handling files newer on destination than on source""" SKIP = 101 #: skip syncing such file REPLACE = 102 #: replace the file on the destination with the (older) file on source RAISE_ERROR = 103 #: raise a non-transient error, failing the sync operation @unique class CompareVersionMode(Enum): """Mode of comparing versions of files to determine what should be synced and what shouldn't""" MODTIME = 201 #: use file modification time on source filesystem SIZE = 202 #: compare using file size NONE = 203 #: compare using file name only class AbstractFileSyncPolicy(metaclass=ABCMeta): """ Abstract policy class. """ DESTINATION_PREFIX = NotImplemented SOURCE_PREFIX = NotImplemented def __init__( self, source_path: AbstractPath | None, source_folder: AbstractFolder, dest_path: AbstractPath | None, dest_folder: AbstractFolder, now_millis: int, keep_days: int, newer_file_mode: NewerFileSyncMode, compare_threshold: int, compare_version_mode: CompareVersionMode = CompareVersionMode.MODTIME, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, upload_mode: UploadMode = UploadMode.FULL, absolute_minimum_part_size: int | None = None, ): """ :param source_path: source file object :param source_folder: source folder object :param dest_path: destination file object :param dest_folder: destination folder object :param now_millis: current time in milliseconds :param keep_days: days to keep before delete :param newer_file_mode: setting which determines handling for destination files newer than on the source :param compare_threshold: when comparing with size or time for sync :param compare_version_mode: how to compare source and destination files :param encryption_settings_provider: encryption setting provider :param upload_mode: file upload mode :param absolute_minimum_part_size: minimum file part size that can be uploaded to the server """ self._source_path = source_path self._source_folder = source_folder self._dest_path = dest_path self._keep_days = keep_days self._newer_file_mode = newer_file_mode self._compare_version_mode = compare_version_mode self._compare_threshold = compare_threshold self._dest_folder = dest_folder self._now_millis = now_millis self._transferred = False self._encryption_settings_provider = encryption_settings_provider self._upload_mode = upload_mode self._absolute_minimum_part_size = absolute_minimum_part_size def _should_transfer(self) -> bool: """ Decide whether to transfer the file from the source to the destination. """ if self._source_path is None or not self._source_path.is_visible(): # No source file. Nothing to transfer. return False elif self._dest_path is None: # Source file exists, but no destination file. Always transfer. return True else: # Both exist. Transfer only if the two are different. return self.files_are_different( self._source_path, self._dest_path, self._compare_threshold, self._compare_version_mode, self._newer_file_mode, ) @classmethod def files_are_different( cls, source_path: AbstractPath, dest_path: AbstractPath, compare_threshold: int | None = None, compare_version_mode: CompareVersionMode = CompareVersionMode.MODTIME, newer_file_mode: NewerFileSyncMode = NewerFileSyncMode.RAISE_ERROR, ): """ Compare two files and determine if the the destination file should be replaced by the source file. :param b2sdk.v2.AbstractPath source_path: source file object :param b2sdk.v2.AbstractPath dest_path: destination file object :param int compare_threshold: compare threshold when comparing by time or size :param b2sdk.v2.CompareVersionMode compare_version_mode: source file version comparator method :param b2sdk.v2.NewerFileSyncMode newer_file_mode: newer destination handling method """ # Optionally set a compare threshold for fuzzy comparison compare_threshold = compare_threshold or 0 # Compare using file name only if compare_version_mode == CompareVersionMode.NONE: return False # Compare using modification time elif compare_version_mode == CompareVersionMode.MODTIME: # Get the modification time of the latest versions source_mod_time = source_path.mod_time dest_mod_time = dest_path.mod_time diff_mod_time = abs(source_mod_time - dest_mod_time) compare_threshold_exceeded = diff_mod_time > compare_threshold logger.debug( 'File %s: source time %s, dest time %s, diff %s, threshold %s, diff > threshold %s', source_path.relative_path, source_mod_time, dest_mod_time, diff_mod_time, compare_threshold, compare_threshold_exceeded, ) if compare_threshold_exceeded: # Source is newer if dest_mod_time < source_mod_time: return True # Source is older elif source_mod_time < dest_mod_time: if newer_file_mode == NewerFileSyncMode.REPLACE: return True elif newer_file_mode == NewerFileSyncMode.SKIP: return False else: raise DestFileNewer( dest_path, source_path, cls.DESTINATION_PREFIX, cls.SOURCE_PREFIX ) # Compare using file size elif compare_version_mode == CompareVersionMode.SIZE: # Get file size of the latest versions source_size = source_path.size dest_size = dest_path.size diff_size = abs(source_size - dest_size) compare_threshold_exceeded = diff_size > compare_threshold logger.debug( 'File %s: source size %s, dest size %s, diff %s, threshold %s, diff > threshold %s', source_path.relative_path, source_size, dest_size, diff_size, compare_threshold, compare_threshold_exceeded, ) # Replace if size difference is over threshold return compare_threshold_exceeded else: raise InvalidArgument('compare_version_mode', 'is invalid option') def get_all_actions(self): """ Yield file actions. """ if self._should_transfer(): yield self._make_transfer_action() self._transferred = True assert self._dest_path is not None or self._source_path is not None yield from self._get_hide_delete_actions() def _get_hide_delete_actions(self): """ Subclass policy can override this to hide or delete files. """ return [] def _get_source_mod_time(self) -> int: if self._source_path is None: return 0 return self._source_path.mod_time @abstractmethod def _make_transfer_action(self): """ Return an action representing transfer of file according to the selected policy. """ class DownPolicy(AbstractFileSyncPolicy): """ File is synced down (from the cloud to disk). """ DESTINATION_PREFIX = 'local://' SOURCE_PREFIX = 'b2://' def _make_transfer_action(self): return B2DownloadAction( cast(B2Path, self._source_path), self._source_folder.make_full_path(self._source_path.relative_path), self._dest_folder.make_full_path(self._source_path.relative_path), self._encryption_settings_provider, ) class UpPolicy(AbstractFileSyncPolicy): """ File is synced up (from disk the cloud). """ DESTINATION_PREFIX = 'b2://' SOURCE_PREFIX = 'local://' def _make_transfer_action(self): # Find out if we want to append with new bytes or replace completely if self._upload_mode == UploadMode.INCREMENTAL and self._dest_path: return B2IncrementalUploadAction( self._source_folder.make_full_path(self._source_path.relative_path), self._source_path.relative_path, self._dest_folder.make_full_path(self._source_path.relative_path), self._get_source_mod_time(), self._source_path.size, self._encryption_settings_provider, cast(B2Path, self._dest_path).selected_version, self._absolute_minimum_part_size, ) else: return B2UploadAction( self._source_folder.make_full_path(self._source_path.relative_path), self._source_path.relative_path, self._dest_folder.make_full_path(self._source_path.relative_path), self._get_source_mod_time(), self._source_path.size, self._encryption_settings_provider, ) class UpAndDeletePolicy(UpPolicy): """ File is synced up (from disk to the cloud) and the delete flag is SET. """ def _get_hide_delete_actions(self): yield from super()._get_hide_delete_actions() yield from make_b2_delete_actions( self._source_path, self._dest_path, self._dest_folder, self._transferred, ) class UpAndKeepDaysPolicy(UpPolicy): """ File is synced up (from disk to the cloud) and the keepDays flag is SET. """ def _get_hide_delete_actions(self): for action in super()._get_hide_delete_actions(): yield action for action in make_b2_keep_days_actions( self._source_path, self._dest_path, self._dest_folder, self._transferred, self._keep_days, self._now_millis, ): yield action class DownAndDeletePolicy(DownPolicy): """ File is synced down (from the cloud to disk) and the delete flag is SET. """ def _get_hide_delete_actions(self): yield from super()._get_hide_delete_actions() if self._dest_path is not None and ( self._source_path is None or not self._source_path.is_visible() ): yield LocalDeleteAction( self._dest_path.relative_path, self._dest_folder.make_full_path(self._dest_path.relative_path), ) class DownAndKeepDaysPolicy(DownPolicy): """ File is synced down (from the cloud to disk) and the keepDays flag is SET. """ pass class CopyPolicy(AbstractFileSyncPolicy): """ File is copied (server-side). """ DESTINATION_PREFIX = 'b2://' SOURCE_PREFIX = 'b2://' def _make_transfer_action(self): return B2CopyAction( self._source_folder.make_full_path(self._source_path.relative_path), cast(B2Path, self._source_path), self._dest_folder.make_full_path(self._source_path.relative_path), cast(B2Folder, self._source_folder).bucket, cast(B2Folder, self._dest_folder).bucket, self._encryption_settings_provider, ) class CopyAndDeletePolicy(CopyPolicy): """ File is copied (server-side) and the delete flag is SET. """ def _get_hide_delete_actions(self): for action in super()._get_hide_delete_actions(): yield action for action in make_b2_delete_actions( self._source_path, self._dest_path, self._dest_folder, self._transferred, ): yield action class CopyAndKeepDaysPolicy(CopyPolicy): """ File is copied (server-side) and the keepDays flag is SET. """ def _get_hide_delete_actions(self): for action in super()._get_hide_delete_actions(): yield action for action in make_b2_keep_days_actions( self._source_path, self._dest_path, self._dest_folder, self._transferred, self._keep_days, self._now_millis, ): yield action def make_b2_delete_note(version, index, transferred): """ Create a note message for delete action. :param b2sdk.v2.FileVersion version: an object which contains file version info :param int index: file version index :param bool transferred: if True, file has been transferred, False otherwise """ note = '' if version.action == 'hide': note = '(hide marker)' elif transferred or 0 < index: note = '(old version)' return note def make_b2_delete_actions( source_path: AbstractPath | None, dest_path: B2Path | None, dest_folder: AbstractFolder, transferred: bool, ): """ Create the actions to delete files stored on B2, which are not present locally. :param source_path: source file object :param dest_path: destination file object :param dest_folder: destination folder :param transferred: if True, file has been transferred, False otherwise """ if dest_path is None: # B2 does not really store folders, so there is no need to hide # them or delete them return for version_index, version in enumerate(dest_path.all_versions): keep = (version_index == 0) and (source_path is not None) and not transferred if not keep: yield B2DeleteAction( dest_path.relative_path, dest_folder.make_full_path(dest_path.relative_path), version.id_, make_b2_delete_note(version, version_index, transferred), ) def make_b2_keep_days_actions( source_path: AbstractPath | None, dest_path: B2Path | None, dest_folder: AbstractFolder, transferred: bool, keep_days: int, now_millis: int, ): """ Create the actions to hide or delete existing versions of a file stored in b2. When keepDays is set, all files that were visible any time from keepDays ago until now must be kept. If versions were uploaded 5 days ago, 15 days ago, and 25 days ago, and the keepDays is 10, only the 25 day-old version can be deleted. The 15 day-old version was visible 10 days ago. :param source_path: source file object :param dest_path: destination file object :param dest_folder: destination folder object :param transferred: if True, file has been transferred, False otherwise :param keep_days: how many days to keep a file :param now_millis: current time in milliseconds """ deleting = False if dest_path is None: # B2 does not really store folders, so there is no need to hide # them or delete them return for version_index, version in enumerate(dest_path.all_versions): # How old is this version? age_days = (now_millis - version.mod_time_millis) / ONE_DAY_IN_MS # Mostly, the versions are ordered by time, newest first, # BUT NOT ALWAYS. The mod time we have is the src_last_modified_millis # from the file info (if present), or the upload start time # (if not present). The user-specified src_last_modified_millis # may not be in order. Because of that, we no longer # assert that age_days is non-decreasing. # # Note that if there is an out-of-order date that is old enough # to trigger deletions, all the versions uploaded before that # (the ones after it in the list) will be deleted, even if they # aren't over the age threshold. # Do we need to hide this version? if version_index == 0 and source_path is None and version.action == 'upload': yield B2HideAction( dest_path.relative_path, dest_folder.make_full_path(dest_path.relative_path) ) # Can we start deleting? Once we start deleting, all older # versions will also be deleted. if version.action == 'hide': if keep_days < age_days: deleting = True # Delete this version if deleting: yield B2DeleteAction( dest_path.relative_path, dest_folder.make_full_path(dest_path.relative_path), version.id_, make_b2_delete_note(version, version_index, transferred), ) # Can we start deleting with the next version, based on the # age of this one? if keep_days < age_days: deleting = True b2-sdk-python-2.8.0/b2sdk/_internal/sync/policy_manager.py000066400000000000000000000104321474454370000234310ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/sync/policy_manager.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from ..scan.folder import AbstractFolder from ..scan.path import AbstractPath from ..transfer.outbound.upload_source import UploadMode from .encryption_provider import AbstractSyncEncryptionSettingsProvider from .policy import ( AbstractFileSyncPolicy, CompareVersionMode, CopyAndDeletePolicy, CopyAndKeepDaysPolicy, CopyPolicy, DownAndDeletePolicy, DownAndKeepDaysPolicy, DownPolicy, NewerFileSyncMode, UpAndDeletePolicy, UpAndKeepDaysPolicy, UpPolicy, ) class SyncPolicyManager: """ Policy manager; implement a logic to get a correct policy class and create a policy object based on various parameters. """ def __init__(self): self.policies = {} # dict<,> def get_policy( self, sync_type: str, source_path: AbstractPath | None, source_folder: AbstractFolder, dest_path: AbstractPath | None, dest_folder: AbstractFolder, now_millis: int, delete: bool, keep_days: int, newer_file_mode: NewerFileSyncMode, compare_threshold: int, compare_version_mode: CompareVersionMode, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider, upload_mode: UploadMode, absolute_minimum_part_size: int, ) -> AbstractFileSyncPolicy: """ Return a policy object. :param sync_type: synchronization type :param source_path: source file :param source_folder: a source folder path :param dest_path: destination file :param dest_folder: a destination folder path :param now_millis: current time in milliseconds :param delete: delete policy :param keep_days: keep for days policy :param newer_file_mode: setting which determines handling for destination files newer than on the source :param compare_threshold: difference between file modification time or file size :param compare_version_mode: setting which determines how to compare source and destination files :param encryption_settings_provider: an object which decides which encryption to use (if any) :param upload_mode: determines how file uploads are handled :param absolute_minimum_part_size: minimum file part size for large files :return: a policy object """ policy_class = self.get_policy_class(sync_type, delete, keep_days) return policy_class( source_path, source_folder, dest_path, dest_folder, now_millis, keep_days, newer_file_mode, compare_threshold, compare_version_mode, encryption_settings_provider, upload_mode, absolute_minimum_part_size, ) def get_policy_class(self, sync_type, delete, keep_days): """ Get policy class by a given sync type. :param str sync_type: synchronization type :param bool delete: if True, delete files and update from source :param int keep_days: keep for `keep_days` before delete :return: a policy class """ if sync_type == 'local-to-b2': if delete: return UpAndDeletePolicy elif keep_days: return UpAndKeepDaysPolicy else: return UpPolicy elif sync_type == 'b2-to-local': if delete: return DownAndDeletePolicy elif keep_days: return DownAndKeepDaysPolicy else: return DownPolicy elif sync_type == 'b2-to-b2': if delete: return CopyAndDeletePolicy elif keep_days: return CopyAndKeepDaysPolicy else: return CopyPolicy raise NotImplementedError( f'invalid sync type: {sync_type}, keep_days: {keep_days}, delete: {delete}' ) POLICY_MANAGER = SyncPolicyManager() b2-sdk-python-2.8.0/b2sdk/_internal/sync/report.py000066400000000000000000000142401474454370000217540ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/sync/report.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging import time import typing from dataclasses import dataclass from ..progress import AbstractProgressListener from ..scan.report import ProgressReport from ..utils import format_and_scale_fraction, format_and_scale_number logger = logging.getLogger(__name__) if typing.TYPE_CHECKING: from io import ( TextIOWrapper, # noqa: F401 # sphinx_autodoc_typehints breaks doc build without this import ) @dataclass class SyncReport(ProgressReport): """ Handle reporting progress for syncing. Print out each file as it is processed, and puts up a sequence of progress bars. The progress bars are: - Step 1/1: count local files - Step 2/2: compare file lists - Step 3/3: transfer files This class is THREAD SAFE, so it can be used from parallel sync threads. """ def __post_init__(self): self.compare_done = False self.compare_count = 0 self.total_transfer_files = 0 # set in end_compare() self.total_transfer_bytes = 0 # set in end_compare() self.transfer_files = 0 self.transfer_bytes = 0 super().__post_init__() def _update_progress(self): if self.closed or self.no_progress: return now = time.time() interval = now - self._last_update_time if interval < self.UPDATE_INTERVAL: return self._last_update_time = now time_delta = now - self.start_time rate = 0 if time_delta == 0 else int(self.transfer_bytes / time_delta) if not self.total_done: message = ' count: %d files compare: %d files updated: %d files %s %s' % ( self.total_count, self.compare_count, self.transfer_files, format_and_scale_number(self.transfer_bytes, 'B'), format_and_scale_number(rate, 'B/s'), ) elif not self.compare_done: message = ' compare: %d/%d files updated: %d files %s %s' % ( self.compare_count, self.total_count, self.transfer_files, format_and_scale_number(self.transfer_bytes, 'B'), format_and_scale_number(rate, 'B/s'), ) else: message = ' compare: %d/%d files updated: %d/%d files %s %s' % ( self.compare_count, self.total_count, self.transfer_files, self.total_transfer_files, format_and_scale_fraction(self.transfer_bytes, self.total_transfer_bytes, 'B'), format_and_scale_number(rate, 'B/s'), ) self._print_line(message, False) def update_compare(self, delta): """ Report that more files have been compared. :param delta: number of files compared :type delta: int """ with self.lock: self.compare_count += delta self._update_progress() def end_compare(self, total_transfer_files, total_transfer_bytes): """ Report that the comparison has been finished. :param total_transfer_files: total number of transferred files :type total_transfer_files: int :param total_transfer_bytes: total number of transferred bytes :type total_transfer_bytes: int """ with self.lock: self.compare_done = True self.total_transfer_files = total_transfer_files self.total_transfer_bytes = total_transfer_bytes self._update_progress() def update_transfer(self, file_delta, byte_delta): """ Update transfer info. :param file_delta: number of files transferred :type file_delta: int :param byte_delta: number of bytes transferred :type byte_delta: int """ with self.lock: self.transfer_files += file_delta self.transfer_bytes += byte_delta self._update_progress() class SyncFileReporter(AbstractProgressListener): """ Listen to the progress for a single file and pass info on to a SyncReporter. """ def __init__(self, reporter, *args, **kwargs): """ :param reporter: a reporter object """ super().__init__(*args, **kwargs) self.bytes_so_far = 0 self.reporter = reporter def close(self): """ Perform a clean-up. """ # no more bytes are done, but the file is done self.reporter.update_transfer(1, 0) def set_total_bytes(self, total_byte_count): """ Set total bytes count. :param total_byte_count: total byte count :type total_byte_count: int """ pass def bytes_completed(self, byte_count): """ Set bytes completed count. :param byte_count: total byte count :type byte_count: int """ self.reporter.update_transfer(0, byte_count - self.bytes_so_far) self.bytes_so_far = byte_count def sample_sync_report_run(): """ Generate a sample report. """ import sys sync_report = SyncReport(sys.stdout, False) for i in range(20): sync_report.update_total(1) time.sleep(0.2) if i == 10: sync_report.print_completion('transferred: a.txt') if i % 2 == 0: sync_report.update_compare(1) sync_report.end_total() for i in range(10): sync_report.update_compare(1) time.sleep(0.2) if i == 3: sync_report.print_completion('transferred: b.txt') if i == 4: sync_report.update_transfer(25, 25000) sync_report.end_compare(50, 50000) for i in range(25): if i % 2 == 0: sync_report.print_completion('transferred: %d.txt' % i) sync_report.update_transfer(1, 1000) time.sleep(0.2) sync_report.close() b2-sdk-python-2.8.0/b2sdk/_internal/sync/sync.py000066400000000000000000000344051474454370000214220ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/sync/sync.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import concurrent.futures as futures import logging from enum import Enum, unique from typing import cast from ..bounded_queue_executor import BoundedQueueExecutor from ..scan.exception import InvalidArgument from ..scan.folder import AbstractFolder, B2Folder, LocalFolder from ..scan.path import AbstractPath from ..scan.policies import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from ..scan.scan import zip_folders from ..transfer.outbound.upload_source import UploadMode from .encryption_provider import ( SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, AbstractSyncEncryptionSettingsProvider, ) from .exception import IncompleteSync from .policy import CompareVersionMode, NewerFileSyncMode from .policy_manager import POLICY_MANAGER, SyncPolicyManager from .report import SyncReport logger = logging.getLogger(__name__) def count_files(local_folder, reporter, policies_manager): """ Count all of the files in a local folder. :param b2sdk._internal.scan.folder.AbstractFolder local_folder: a folder object. :param reporter: reporter object """ # Don't pass in a reporter to all_files. Broken symlinks will be reported # during the next pass when the source and dest files are compared. for _ in local_folder.all_files(None, policies_manager=policies_manager): reporter.update_total(1) reporter.end_total() @unique class KeepOrDeleteMode(Enum): """Mode of dealing with old versions of files on the destination""" DELETE = 301 #: delete the old version as soon as the new one has been uploaded KEEP_BEFORE_DELETE = 302 #: keep the old versions of the file for a configurable number of days before deleting them, always keeping the newest version NO_DELETE = 303 #: keep old versions of the file, do not delete anything class Synchronizer: """ Copies multiple "files" from source to destination. Optionally deletes or hides destination files that the source does not have. The synchronizer can copy files: - From a B2 bucket to a local destination. - From a local source to a B2 bucket. - From one B2 bucket to another. - Between different folders in the same B2 bucket. It will sync only the latest versions of files. By default, the synchronizer: - Fails when the specified source directory doesn't exist or is empty. (see ``allow_empty_source`` argument) - Fails when the source is newer. (see ``newer_file_mode`` argument) - Doesn't delete a file if it's present on the destination but not on the source. (see ``keep_days_or_delete`` and ``keep_days`` arguments) - Compares files based on modification time. (see ``compare_version_mode`` and ``compare_threshold`` arguments) """ def __init__( self, max_workers, policies_manager=DEFAULT_SCAN_MANAGER, dry_run=False, allow_empty_source=False, newer_file_mode=NewerFileSyncMode.RAISE_ERROR, keep_days_or_delete=KeepOrDeleteMode.NO_DELETE, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=None, keep_days=None, sync_policy_manager: SyncPolicyManager = POLICY_MANAGER, upload_mode: UploadMode = UploadMode.FULL, absolute_minimum_part_size: int | None = None, ): """ Initialize synchronizer class and validate arguments :param int max_workers: max number of workers :param policies_manager: object which decides which files to process :param bool dry_run: test mode, does not actually transfer/delete when enabled :param bool allow_empty_source: if True, do not check whether source folder is empty :param b2sdk.v2.NewerFileSyncMode newer_file_mode: setting which determines handling for destination files newer than on the source :param b2sdk.v2.KeepOrDeleteMode keep_days_or_delete: setting which determines if we should delete or not delete or keep for `keep_days` :param b2sdk.v2.CompareVersionMode compare_version_mode: how to compare the source and destination files to find new ones :param int compare_threshold: should be greater than 0, default is 0 :param int keep_days: if keep_days_or_delete is `b2sdk.v2.KeepOrDeleteMode.KEEP_BEFORE_DELETE`, then this should be greater than 0 :param SyncPolicyManager sync_policy_manager: object which decides what to do with each file (upload, download, delete, copy, hide etc) :param b2sdk.v2.UploadMode upload_mode: determines how file uploads are handled :param int absolute_minimum_part_size: minimum file part size for large files """ self.newer_file_mode = newer_file_mode self.keep_days_or_delete = keep_days_or_delete self.keep_days = keep_days self.compare_version_mode = compare_version_mode self.compare_threshold = compare_threshold or 0 self.dry_run = dry_run self.allow_empty_source = allow_empty_source self.policies_manager = ( policies_manager # actually it should be called scan_policies_manager ) self.sync_policy_manager = sync_policy_manager self.max_workers = max_workers self.upload_mode = upload_mode self.absolute_minimum_part_size = absolute_minimum_part_size self._validate() def _validate(self): if self.compare_threshold < 0: raise InvalidArgument('compare_threshold', 'must be a positive integer') if self.newer_file_mode not in tuple(NewerFileSyncMode): raise InvalidArgument( 'newer_file_mode', 'must be one of :%s' % NewerFileSyncMode.__members__, ) if self.keep_days_or_delete not in tuple(KeepOrDeleteMode): raise InvalidArgument( 'keep_days_or_delete', 'must be one of :%s' % KeepOrDeleteMode.__members__, ) if ( self.keep_days_or_delete == KeepOrDeleteMode.KEEP_BEFORE_DELETE and self.keep_days is None ): raise InvalidArgument( 'keep_days', 'is required when keep_days_or_delete is %s' % KeepOrDeleteMode.KEEP_BEFORE_DELETE, ) if self.compare_version_mode not in tuple(CompareVersionMode): raise InvalidArgument( 'compare_version_mode', 'must be one of :%s' % CompareVersionMode.__members__, ) def sync_folders( self, source_folder: AbstractFolder, dest_folder: AbstractFolder, now_millis: int, reporter: SyncReport | None, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ Syncs two folders. Always ensures that every file in the source is also in the destination. Deletes any file versions in the destination older than history_days. :param source_folder: source folder object :param dest_folder: destination folder object :param now_millis: current time in milliseconds :param reporter: progress reporter :param encryption_settings_provider: encryption setting provider """ source_type = source_folder.folder_type() dest_type = dest_folder.folder_type() if source_type != 'b2' and dest_type != 'b2': raise ValueError('Sync between two local folders is not supported!') # For downloads, make sure that the target directory is there. if dest_type == 'local' and not self.dry_run: cast(LocalFolder, dest_folder).ensure_present() if source_type == 'local' and not self.allow_empty_source: cast(LocalFolder, source_folder).ensure_non_empty() # Make an executor to count files and run all of the actions. This is # not the same as the executor in the API object which is used for # uploads. The tasks in this executor wait for uploads. Putting them # in the same thread pool could lead to deadlock. # # We use an executor with a bounded queue to avoid using up lots of memory # when syncing lots of files. unbounded_executor = futures.ThreadPoolExecutor(max_workers=self.max_workers) queue_limit = self.max_workers + 1000 sync_executor = BoundedQueueExecutor(unbounded_executor, queue_limit=queue_limit) if source_type == 'local' and reporter is not None: # Start the thread that counts the local files. That's the operation # that should be fastest, and it provides scale for the progress reporting. sync_executor.submit(count_files, source_folder, reporter, self.policies_manager) # Bucket for scheduling actions. # For bucket-to-bucket sync, the bucket for the API calls should be the destination. action_bucket = None if dest_type == 'b2': action_bucket = cast(B2Folder, dest_folder).bucket elif source_type == 'b2': action_bucket = cast(B2Folder, source_folder).bucket # Schedule each of the actions. for action in self._make_folder_sync_actions( source_folder, dest_folder, now_millis, reporter, self.policies_manager, encryption_settings_provider, ): logging.debug('scheduling action %s on bucket %s', action, action_bucket) sync_executor.submit(action.run, action_bucket, reporter, self.dry_run) # Wait for everything to finish sync_executor.shutdown() if sync_executor.get_num_exceptions() != 0: raise IncompleteSync('sync is incomplete') def _make_folder_sync_actions( self, source_folder: AbstractFolder, dest_folder: AbstractFolder, now_millis: int, reporter: SyncReport, policies_manager: ScanPoliciesManager = DEFAULT_SCAN_MANAGER, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ Yield a sequence of actions that will sync the destination folder to the source folder. :param source_folder: source folder object :param dest_folder: destination folder object :param now_millis: current time in milliseconds :param reporter: reporter object :param policies_manager: object which decides which files to process :param encryption_settings_provider: encryption setting provider """ if ( self.keep_days_or_delete == KeepOrDeleteMode.KEEP_BEFORE_DELETE and dest_folder.folder_type() == 'local' ): raise InvalidArgument('keep_days_or_delete', 'cannot be used for local files') source_type = source_folder.folder_type() dest_type = dest_folder.folder_type() sync_type = f'{source_type}-to-{dest_type}' if source_type != 'b2' and dest_type != 'b2': raise ValueError('Sync between two local folders is not supported!') total_files = 0 total_bytes = 0 for source_path, dest_path in zip_folders( source_folder, dest_folder, reporter, policies_manager, ): if source_path is None: logger.debug('determined that %s is not present on source', dest_path) elif dest_path is None: logger.debug('determined that %s is not present on destination', source_path) if source_path is not None: if source_type == 'b2': # For buckets we don't want to count files separately as it would require # more API calls. Instead, we count them when comparing. reporter.update_total(1) reporter.update_compare(1) for action in self._make_file_sync_actions( sync_type, source_path, dest_path, source_folder, dest_folder, now_millis, encryption_settings_provider, ): total_files += 1 total_bytes += action.get_bytes() yield action if reporter is not None: if source_type == 'b2': # For buckets we don't want to count files separately as it would require # more API calls. Instead, we count them when comparing. reporter.end_total() reporter.end_compare(total_files, total_bytes) def _make_file_sync_actions( self, sync_type: str, source_path: AbstractPath | None, dest_path: AbstractPath | None, source_folder: AbstractFolder, dest_folder: AbstractFolder, now_millis: int, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ Yields the sequence of actions needed to sync the two files :param sync_type: synchronization type :param source_path: source file object :param dest_path: destination file object :param source_folder: a source folder object :param dest_folder: a destination folder object :param now_millis: current time in milliseconds :param encryption_settings_provider: encryption setting provider """ delete = self.keep_days_or_delete == KeepOrDeleteMode.DELETE policy = self.sync_policy_manager.get_policy( sync_type, source_path, source_folder, dest_path, dest_folder, now_millis, delete, self.keep_days, self.newer_file_mode, self.compare_threshold, self.compare_version_mode, encryption_settings_provider=encryption_settings_provider, upload_mode=self.upload_mode, absolute_minimum_part_size=self.absolute_minimum_part_size, ) return policy.get_all_actions() b2-sdk-python-2.8.0/b2sdk/_internal/transfer/000077500000000000000000000000001474454370000207365ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/transfer/__init__.py000066400000000000000000000011541474454370000230500ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from .inbound.download_manager import DownloadManager from .outbound.copy_manager import CopyManager from .outbound.upload_manager import UploadManager from .emerge.emerger import Emerger __all__ = [ 'DownloadManager', 'CopyManager', 'UploadManager', 'Emerger', ] b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/000077500000000000000000000000001474454370000222025ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/__init__.py000066400000000000000000000005311474454370000243120ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/emerge/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/emerger.py000066400000000000000000000347251474454370000242150ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/emerge/emerger.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from typing import Iterator from b2sdk._internal.encryption.setting import EncryptionSetting from b2sdk._internal.file_lock import FileRetentionSetting, LegalHold from b2sdk._internal.http_constants import LARGE_FILE_SHA1 from b2sdk._internal.progress import AbstractProgressListener from b2sdk._internal.transfer.emerge.executor import EmergeExecutor from b2sdk._internal.transfer.emerge.planner.planner import EmergePlan, EmergePlanner from b2sdk._internal.transfer.emerge.write_intent import WriteIntent from b2sdk._internal.utils import B2TraceMetaAbstract, Sha1HexDigest, iterator_peek logger = logging.getLogger(__name__) class Emerger(metaclass=B2TraceMetaAbstract): """ Handle complex actions around multi source copy/uploads. This class can be used to build advanced copy workflows like incremental upload. It creates a emerge plan and pass it to emerge executor - all complex logic is actually implemented in :class:`b2sdk._internal.transfer.emerge.planner.planner.EmergePlanner` and :class:`b2sdk._internal.transfer.emerge.executor.EmergeExecutor` """ DEFAULT_STREAMING_MAX_QUEUE_SIZE = 100 def __init__(self, services): """ :param b2sdk.v2.Services services: """ self.services = services self.emerge_executor = EmergeExecutor(services) @classmethod def _get_updated_file_info_with_large_file_sha1( cls, file_info: dict[str, str] | None, write_intents: list[WriteIntent] | None, emerge_plan: EmergePlan, large_file_sha1: Sha1HexDigest | None = None, ) -> dict[str, str] | None: if not emerge_plan.is_large_file(): # Emerge plan doesn't construct a large file, no point setting the large_file_sha1 return file_info file_sha1 = large_file_sha1 if not file_sha1 and write_intents is not None and len(write_intents) == 1: # large_file_sha1 was not given explicitly, but there's just one write intent, perhaps it has a hash file_sha1 = write_intents[0].get_content_sha1() out_file_info = file_info if file_sha1: out_file_info = dict(file_info) if file_info else {} out_file_info[LARGE_FILE_SHA1] = file_sha1 return out_file_info def _emerge( self, emerge_function, bucket_id, write_intents_iterable, file_name, content_type, file_info, progress_listener, recommended_upload_part_size=None, continue_large_file_id=None, max_queue_size=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, min_part_size: int | None = None, max_part_size: int | None = None, large_file_sha1: Sha1HexDigest | None = None, check_first_intent_for_sha1: bool = True, custom_upload_timestamp: int | None = None, ): planner = self.get_emerge_planner( min_part_size=min_part_size, recommended_upload_part_size=recommended_upload_part_size, max_part_size=max_part_size, ) # Large file SHA1 operation, possibly on intents. large_file_sha1_intents_for_check = None all_write_intents = write_intents_iterable if check_first_intent_for_sha1: write_intents_iterator = iter(all_write_intents) large_file_sha1_intents_for_check, all_write_intents = iterator_peek( write_intents_iterator, 2 ) emerge_plan = emerge_function(planner, all_write_intents) out_file_info = self._get_updated_file_info_with_large_file_sha1( file_info, large_file_sha1_intents_for_check, emerge_plan, large_file_sha1, ) return self.emerge_executor.execute_emerge_plan( emerge_plan, bucket_id, file_name, content_type, out_file_info, progress_listener, continue_large_file_id=continue_large_file_id, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, # Max queue size is only used in case of large files. # Passing anything for small files does nothing. max_queue_size=max_queue_size, custom_upload_timestamp=custom_upload_timestamp, ) def emerge( self, bucket_id: str, write_intents: list[WriteIntent], file_name: str, content_type: str | None, file_info: dict[str, str] | None, progress_listener: AbstractProgressListener, recommended_upload_part_size: int | None = None, continue_large_file_id: str | None = None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, min_part_size: int | None = None, max_part_size: int | None = None, large_file_sha1: Sha1HexDigest | None = None, custom_upload_timestamp: int | None = None, ): """ Create a new file (object in the cloud, really) from an iterable (list, tuple etc) of write intents. :param bucket_id: a bucket ID :param write_intents: write intents to process to create a file :param file_name: the file name of the new B2 file :param content_type: the MIME type or ``None`` to determine automatically :param file_info: a file info to store with the file or ``None`` to not store anything :param progress_listener: a progress listener object to use :param recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, if ``None`` in multipart case it would always start a new large file :param encryption: encryption settings (``None`` if unknown) :param file_retention: file retention setting :param legal_hold: legal hold setting :param min_part_size: lower limit of part size for the transfer planner, in bytes :param max_part_size: upper limit of part size for the transfer planner, in bytes :param large_file_sha1: SHA1 for this file, if ``None`` and there's exactly one intent, it'll be taken from it :param custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch """ return self._emerge( EmergePlanner.get_emerge_plan, bucket_id, write_intents, file_name, content_type, file_info, progress_listener, continue_large_file_id=continue_large_file_id, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, recommended_upload_part_size=recommended_upload_part_size, min_part_size=min_part_size, max_part_size=max_part_size, large_file_sha1=large_file_sha1, custom_upload_timestamp=custom_upload_timestamp, ) def emerge_stream( self, bucket_id: str, write_intent_iterator: Iterator[WriteIntent], file_name: str, content_type: str | None, file_info: dict[str, str] | None, progress_listener: AbstractProgressListener, recommended_upload_part_size: int | None = None, continue_large_file_id: str | None = None, max_queue_size: int = DEFAULT_STREAMING_MAX_QUEUE_SIZE, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, min_part_size: int | None = None, max_part_size: int | None = None, large_file_sha1: Sha1HexDigest | None = None, custom_upload_timestamp: int | None = None, ): """ Create a new file (object in the cloud, really) from a stream of write intents. :param bucket_id: a bucket ID :param write_intent_iterator: iterator of :class:`~b2sdk.v2.WriteIntent` :param file_name: the file name of the new B2 file :param content_type: the MIME type or ``None`` to determine automatically :param file_info: a file info to store with the file or ``None`` to not store anything :param progress_listener: a progress listener object to use :param recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, if ``None`` in multipart case it would always start a new large file :param max_queue_size: parallelization level :param encryption: encryption settings (``None`` if unknown) :param file_retention: file retention setting :param legal_hold: legal hold setting :param min_part_size: lower limit of part size for the transfer planner, in bytes :param max_part_size: upper limit of part size for the transfer planner, in bytes :param large_file_sha1: SHA1 for this file, if ``None`` and there's exactly one intent, it'll be taken from it :param custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch """ return self._emerge( EmergePlanner.get_streaming_emerge_plan, bucket_id, write_intent_iterator, file_name, content_type, file_info, progress_listener, continue_large_file_id=continue_large_file_id, max_queue_size=max_queue_size, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, recommended_upload_part_size=recommended_upload_part_size, min_part_size=min_part_size, max_part_size=max_part_size, large_file_sha1=large_file_sha1, custom_upload_timestamp=custom_upload_timestamp, ) def emerge_unbound( self, bucket_id: str, write_intent_iterator: Iterator[WriteIntent], file_name: str, content_type: str | None, file_info: dict[str, str] | None, progress_listener: AbstractProgressListener, recommended_upload_part_size: int | None = None, continue_large_file_id: str | None = None, max_queue_size: int = 1, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, min_part_size: int | None = None, max_part_size: int | None = None, large_file_sha1: Sha1HexDigest | None = None, custom_upload_timestamp: int | None = None, ): """ Create a new file (object in the cloud, really) from an unbound stream of write intents. :param bucket_id: a bucket ID :param write_intent_iterator: iterator of :class:`~b2sdk.v2.WriteIntent` :param file_name: the file name of the new B2 file :param content_type: the MIME type or ``None`` to determine automatically :param file_info: a file info to store with the file or ``None`` to not store anything :param progress_listener: a progress listener object to use :param recommended_upload_part_size: the recommended part size to use for uploading local sources or ``None`` to determine automatically, but remote sources would be copied with maximum possible part size :param continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, if ``None`` in multipart case it would always start a new large file :param max_queue_size: parallelization level, should be equal to the number of buffers available in parallel :param encryption: encryption settings (``None`` if unknown) :param file_retention: file retention setting :param legal_hold: legal hold setting :param min_part_size: lower limit of part size for the transfer planner, in bytes :param max_part_size: upper limit of part size for the transfer planner, in bytes :param large_file_sha1: SHA1 for this file, if ``None`` it's left unset :param custom_upload_timestamp: override object creation date, expressed as a number of milliseconds since epoch """ return self._emerge( EmergePlanner.get_unbound_emerge_plan, bucket_id, write_intent_iterator, file_name, content_type, file_info, progress_listener, continue_large_file_id=continue_large_file_id, max_queue_size=max_queue_size, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, recommended_upload_part_size=recommended_upload_part_size, min_part_size=min_part_size, max_part_size=max_part_size, large_file_sha1=large_file_sha1, check_first_intent_for_sha1=False, custom_upload_timestamp=custom_upload_timestamp, ) def get_emerge_planner( self, recommended_upload_part_size: int | None = None, min_part_size: int | None = None, max_part_size: int | None = None, ): return EmergePlanner.from_account_info( self.services.session.account_info, min_part_size=min_part_size, recommended_upload_part_size=recommended_upload_part_size, max_part_size=max_part_size, ) b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/exception.py000066400000000000000000000010551474454370000245530ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/emerge/exception.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal.exception import B2SimpleError class UnboundStreamBufferTimeout(B2SimpleError): """ Raised when there is no space for a new buffer for a certain amount of time. """ pass b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/executor.py000066400000000000000000000710451474454370000244210ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/emerge/executor.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging import threading from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING from b2sdk._internal.encryption.setting import EncryptionSetting from b2sdk._internal.exception import MaxFileSizeExceeded from b2sdk._internal.file_lock import NO_RETENTION_FILE_SETTING, FileRetentionSetting, LegalHold from b2sdk._internal.http_constants import LARGE_FILE_SHA1 from b2sdk._internal.transfer.outbound.large_file_upload_state import LargeFileUploadState from b2sdk._internal.transfer.outbound.upload_source import UploadSourceStream AUTO_CONTENT_TYPE = 'b2/x-auto' logger = logging.getLogger(__name__) if TYPE_CHECKING: from b2sdk._internal.transfer.emerge.planner.part_definition import UploadEmergePartDefinition from b2sdk._internal.transfer.emerge.planner.planner import StreamingEmergePlan class EmergeExecutor: def __init__(self, services): self.services = services def execute_emerge_plan( self, emerge_plan, bucket_id, file_name, content_type, file_info, progress_listener, continue_large_file_id=None, max_queue_size=None, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): if emerge_plan.is_large_file(): execution = LargeFileEmergeExecution( self.services, bucket_id, file_name, content_type, file_info, progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, continue_large_file_id=continue_large_file_id, max_queue_size=max_queue_size, custom_upload_timestamp=custom_upload_timestamp, ) else: if continue_large_file_id is not None: raise ValueError('Cannot resume emerging single part plan.') execution = SmallFileEmergeExecution( self.services, bucket_id, file_name, content_type, file_info, progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) return execution.execute_plan(emerge_plan) class BaseEmergeExecution(metaclass=ABCMeta): DEFAULT_CONTENT_TYPE = AUTO_CONTENT_TYPE def __init__( self, services, bucket_id, file_name, content_type, file_info, progress_listener, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): self.services = services self.bucket_id = bucket_id self.file_name = file_name self.content_type = content_type self.file_info = file_info self.progress_listener = progress_listener self.encryption = encryption self.file_retention = file_retention self.legal_hold = legal_hold self.custom_upload_timestamp = custom_upload_timestamp @abstractmethod def execute_plan(self, emerge_plan): pass class SmallFileEmergeExecution(BaseEmergeExecution): def execute_plan(self, emerge_plan): emerge_parts = list(emerge_plan.emerge_parts) assert len(emerge_parts) == 1 emerge_part = emerge_parts[0] execution_step_factory = SmallFileEmergeExecutionStepFactory(self, emerge_part) execution_step = execution_step_factory.get_execution_step() future = execution_step.execute() return future.result() class LargeFileEmergeExecution(BaseEmergeExecution): MAX_LARGE_FILE_SIZE = 10 * 1000 * 1000 * 1000 * 1000 # 10 TB def __init__( self, services, bucket_id, file_name, content_type, file_info, progress_listener, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, continue_large_file_id=None, max_queue_size=None, custom_upload_timestamp: int | None = None, ): super().__init__( services, bucket_id, file_name, content_type, file_info, progress_listener, encryption=encryption, file_retention=file_retention, legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) self.continue_large_file_id = continue_large_file_id self.max_queue_size = max_queue_size self._semaphore = None if self.max_queue_size is not None: self._semaphore = threading.Semaphore(self.max_queue_size) def execute_plan(self, emerge_plan: StreamingEmergePlan): total_length = emerge_plan.get_total_length() encryption = self.encryption if total_length is not None and total_length > self.MAX_LARGE_FILE_SIZE: raise MaxFileSizeExceeded(total_length, self.MAX_LARGE_FILE_SIZE) plan_id = emerge_plan.get_plan_id() file_info = dict(self.file_info) if self.file_info is not None else {} if plan_id is not None: file_info['plan_id'] = plan_id self.progress_listener.set_total_bytes(total_length or 0) emerge_parts_dict = None if total_length is not None: emerge_parts_dict = dict(emerge_plan.enumerate_emerge_parts()) unfinished_file, finished_parts = self._get_unfinished_file_and_parts( self.bucket_id, self.file_name, file_info, self.continue_large_file_id, encryption=encryption, file_retention=self.file_retention, legal_hold=self.legal_hold, emerge_parts_dict=emerge_parts_dict, custom_upload_timestamp=self.custom_upload_timestamp, ) if unfinished_file is None: if self.content_type is None: content_type = self.DEFAULT_CONTENT_TYPE else: content_type = self.content_type unfinished_file = self.services.large_file.start_large_file( self.bucket_id, self.file_name, content_type, file_info, encryption=encryption, file_retention=self.file_retention, legal_hold=self.legal_hold, ) file_id = unfinished_file.file_id large_file_upload_state = LargeFileUploadState(self.progress_listener) part_futures = [] for part_number, emerge_part in emerge_plan.enumerate_emerge_parts(): execution_step_factory = LargeFileEmergeExecutionStepFactory( self, emerge_part, part_number, file_id, large_file_upload_state, finished_parts=finished_parts, # it already knows encryption from BaseMergeExecution being passed as self ) execution_step = execution_step_factory.get_execution_step() future = self._execute_step(execution_step) part_futures.append(future) # Collect the sha1 checksums of the parts as the uploads finish. # If any of them raised an exception, that same exception will # be raised here by result() part_sha1_array = [f.result()['contentSha1'] for f in part_futures] # Finish the large file response = self.services.session.finish_large_file(file_id, part_sha1_array) return self.services.api.file_version_factory.from_api_response(response) def _execute_step(self, execution_step: UploadPartExecutionStep): semaphore = self._semaphore if semaphore is None: return execution_step.execute() else: semaphore.acquire() try: future = execution_step.execute() except: semaphore.release() raise else: future.add_done_callback(lambda f: semaphore.release()) return future def _get_unfinished_file_and_parts( self, bucket_id, file_name, file_info, continue_large_file_id, encryption: EncryptionSetting, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, emerge_parts_dict=None, custom_upload_timestamp: int | None = None, ): if 'listFiles' not in self.services.session.account_info.get_allowed()['capabilities']: return None, {} unfinished_file = None finished_parts = {} if continue_large_file_id is not None: unfinished_file = self.services.large_file.get_unfinished_large_file( bucket_id, continue_large_file_id, prefix=file_name, ) if unfinished_file.file_info != file_info: raise ValueError( 'Cannot manually resume unfinished large file with different file_info' ) finished_parts = { part.part_number: part for part in self.services.large_file.list_parts(continue_large_file_id) } elif 'plan_id' in file_info: assert emerge_parts_dict is not None unfinished_file, finished_parts = self._find_unfinished_file_by_plan_id( bucket_id, file_name, file_info, emerge_parts_dict, encryption, file_retention, legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) elif emerge_parts_dict is not None: unfinished_file, finished_parts = self._match_unfinished_file_if_possible( bucket_id, file_name, file_info, emerge_parts_dict, encryption, file_retention, legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) return unfinished_file, finished_parts def _find_matching_unfinished_file( self, bucket_id, file_name, file_info, emerge_parts_dict, encryption: EncryptionSetting, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, check_file_info_without_large_file_sha1: bool | None = False, eager_mode: bool | None = False, ): """ Search for a matching unfinished large file in the specified bucket. In case a matching file is found but has inconsistencies (for example, mismatching file info or encryption settings), mismatches are logged. :param bucket_id: The identifier of the bucket where the unfinished file resides. :param file_name: The name of the file to be matched. :param file_info: Information about the file to be uploaded. :param emerge_parts_dict: A dictionary containing the parts of the file to be emerged. :param encryption: The encryption settings for the file. :param file_retention: The retention settings for the file, if any. :param legal_hold: The legal hold status of the file, if any. :param custom_upload_timestamp: The custom timestamp for the upload, if any. :param check_file_info_without_large_file_sha1: A flag indicating whether the file information should be checked without the `large_file_sha1`. :param eager_mode: A flag indicating whether the first matching file should be returned. :return: A tuple of the best matching unfinished file and its finished parts. If no match is found, returns `None`. """ file_retention = file_retention or NO_RETENTION_FILE_SETTING best_match_file = None best_match_parts = {} best_match_parts_len = 0 for file_ in self.services.large_file.list_unfinished_large_files( bucket_id, prefix=file_name ): if file_.file_name != file_name: logger.debug('Rejecting %s: file name mismatch', file_.file_id) continue if file_.file_info != file_info: if check_file_info_without_large_file_sha1: file_info_without_large_file_sha1 = self._get_file_info_without_large_file_sha1( file_info ) if ( file_info_without_large_file_sha1 != self._get_file_info_without_large_file_sha1(file_.file_info) ): logger.debug( 'Rejecting %s: file info mismatch after dropping `large_file_sha1`', file_.file_id, ) continue else: logger.debug('Rejecting %s: file info mismatch', file_.file_id) continue if encryption is not None and encryption != file_.encryption: logger.debug('Rejecting %s: encryption mismatch', file_.file_id) continue if legal_hold is None: if LegalHold.UNSET != file_.legal_hold: logger.debug('Rejecting %s: legal hold mismatch (not unset)', file_.file_id) continue elif legal_hold != file_.legal_hold: logger.debug('Rejecting %s: legal hold mismatch', file_.file_id) continue if file_retention != file_.file_retention: logger.debug('Rejecting %s: retention mismatch', file_.file_id) continue if ( custom_upload_timestamp is not None and file_.upload_timestamp != custom_upload_timestamp ): logger.debug('Rejecting %s: custom_upload_timestamp mismatch', file_.file_id) continue finished_parts = {} conflict_detected = False for part in self.services.large_file.list_parts(file_.file_id): emerge_part = emerge_parts_dict.get(part.part_number) if emerge_part is None: # something is wrong - we have a part that we don't know about # so we can't resume this upload logger.debug( 'Rejecting %s: part %s not found in emerge parts, giving up.', file_.file_id, part.part_number, ) conflict_detected = True break # Compare part sizes if emerge_part.get_length() != part.content_length: logger.debug( 'Rejecting %s: part %s size mismatch', file_.file_id, part.part_number ) conflict_detected = True break # part size doesn't match - so we reupload # Compare part hashes if emerge_part.is_hashable() and emerge_part.get_sha1() != part.content_sha1: logger.debug( 'Rejecting %s: part %s sha1 mismatch', file_.file_id, part.part_number ) conflict_detected = True break # part.sha1 doesn't match - so we reupload finished_parts[part.part_number] = part if conflict_detected: continue finished_parts_len = len(finished_parts) if best_match_file is None or finished_parts_len > best_match_parts_len: best_match_file = file_ best_match_parts = finished_parts best_match_parts_len = finished_parts_len if eager_mode and best_match_file is not None: break return best_match_file, best_match_parts def _find_unfinished_file_by_plan_id( self, bucket_id, file_name, file_info, emerge_parts_dict, encryption: EncryptionSetting, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): """ Search for a matching unfinished large file by plan_id in the specified bucket. This function aims to locate a matching unfinished large file using the plan_id and the supplied parameters. It's used to resume an interrupted upload, centralizing the shared logic between `_find_unfinished_file_by_plan_id` and `_match_unfinished_file_if_possible`. In case a matching file is found but has inconsistencies (for example, mismatching file info or encryption settings), the function checks if 'plan_id' is in file_info, as this is a prerequisite. :param bucket_id: The identifier of the bucket where the unfinished file resides. :param file_name: The name of the file to be matched. :param file_info: Information about the file to be uploaded. :param emerge_parts_dict: A dictionary containing the parts of the file to be emerged. :param encryption: The encryption settings for the file. :param file_retention: The retention settings for the file, if any. :param legal_hold: The legal hold status of the file, if any. :param custom_upload_timestamp: The custom timestamp for the upload, if any. :return: A tuple of the best matching unfinished file and its finished parts. If no match is found, it returns `None`. """ if 'plan_id' not in file_info: raise ValueError("The 'plan_id' key must be in file_info dictionary.") return self._find_matching_unfinished_file( bucket_id=bucket_id, file_name=file_name, file_info=file_info, emerge_parts_dict=emerge_parts_dict, encryption=encryption, file_retention=file_retention or NO_RETENTION_FILE_SETTING, legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, check_file_info_without_large_file_sha1=False, ) @classmethod def _get_file_info_without_large_file_sha1( cls, file_info: dict[str, str] | None, ) -> dict[str, str] | None: if not file_info or LARGE_FILE_SHA1 not in file_info: return file_info out_file_info = dict(file_info) del out_file_info[LARGE_FILE_SHA1] return out_file_info def _match_unfinished_file_if_possible( self, bucket_id, file_name, file_info, emerge_parts_dict, encryption: EncryptionSetting, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): """ Scan for a suitable unfinished large file in the specified bucket to resume upload. This function examines each unfinished large file for a possible match with the provided parameters. This enables resumption of an interrupted upload by reusing the unfinished file, provided that file's info and additional parameters match. Along with the filename and file info, additional parameters like encryption, file retention, legal hold, custom upload timestamp, and cache control are compared for a match. The 'emerge_parts_dict' is also cross-checked for matching file parts. Function is eager to find a match, and will return the first match it finds. If no match is found, it returns `None`. :param bucket_id: The identifier of the bucket containing the unfinished file. :param file_name: The name of the file to find. :param file_info: Information about the file to be uploaded. :param emerge_parts_dict: A dictionary of the parts of the file to be emerged. :param encryption: The encryption settings for the file. :param file_retention: The retention settings for the file, if applicable. :param legal_hold: The legal hold status of the file, if applicable. :param custom_upload_timestamp: The custom timestamp for the upload, if set. :return: A tuple of the best matching unfinished file and its finished parts. If no match is found, returns `None`. """ logger.debug('Checking for matching unfinished large files for %s...', file_name) file_, finished_parts = self._find_matching_unfinished_file( bucket_id, file_name, file_info, emerge_parts_dict, encryption, file_retention, legal_hold, custom_upload_timestamp, check_file_info_without_large_file_sha1=True, eager_mode=True, ) if file_ is None: logger.debug('No matching unfinished files found.') return None, {} logger.debug( 'Unfinished file %s matches with %i finished parts', file_.file_id, len(finished_parts) ) return file_, finished_parts class BaseExecutionStepFactory(metaclass=ABCMeta): def __init__(self, emerge_execution, emerge_part): self.emerge_execution = emerge_execution self.emerge_part = emerge_part @abstractmethod def create_copy_execution_step(self, copy_range): pass @abstractmethod def create_upload_execution_step(self, stream_opener, stream_length=None, stream_sha1=None): pass def get_execution_step(self): return self.emerge_part.get_execution_step(self) class SmallFileEmergeExecutionStepFactory(BaseExecutionStepFactory): def create_copy_execution_step(self, copy_range): return CopyFileExecutionStep(self.emerge_execution, copy_range) def create_upload_execution_step(self, stream_opener, stream_length=None, stream_sha1=None): return UploadFileExecutionStep( self.emerge_execution, stream_opener, stream_length=stream_length, stream_sha1=stream_sha1, ) class LargeFileEmergeExecutionStepFactory(BaseExecutionStepFactory): def __init__( self, emerge_execution, emerge_part: UploadEmergePartDefinition, part_number, large_file_id, large_file_upload_state, finished_parts=None, ): super().__init__(emerge_execution, emerge_part) self.part_number = part_number self.large_file_id = large_file_id self.large_file_upload_state = large_file_upload_state self.finished_parts = finished_parts or {} def create_copy_execution_step(self, copy_range): return CopyPartExecutionStep( self.emerge_execution, copy_range, self.part_number, self.large_file_id, self.large_file_upload_state, ) def create_upload_execution_step(self, stream_opener, stream_length=None, stream_sha1=None): return UploadPartExecutionStep( self.emerge_execution, stream_opener, self.part_number, self.large_file_id, self.large_file_upload_state, stream_length=stream_length, stream_sha1=stream_sha1, finished_parts=self.finished_parts, ) class BaseExecutionStep(metaclass=ABCMeta): def __init__(self, emerge_execution: BaseEmergeExecution): self.emerge_execution = emerge_execution @abstractmethod def execute(self): pass class CopyFileExecutionStep(BaseExecutionStep): def __init__(self, emerge_execution, copy_source_range): super().__init__(emerge_execution) self.copy_source_range = copy_source_range def execute(self): execution = self.emerge_execution # if content type is not None then we support empty dict as default file info # but if content type is None, then setting empty dict as file info # would result with an error, because default in such case is: copy from source if execution.content_type is not None: file_info = execution.file_info or {} else: file_info = None return execution.services.copy_manager.copy_file( self.copy_source_range, execution.file_name, content_type=execution.content_type, file_info=file_info, destination_bucket_id=execution.bucket_id, progress_listener=execution.progress_listener, destination_encryption=execution.encryption, source_encryption=self.copy_source_range.encryption, file_retention=execution.file_retention, legal_hold=execution.legal_hold, ) class CopyPartExecutionStep(BaseExecutionStep): def __init__( self, emerge_execution, copy_source_range, part_number, large_file_id, large_file_upload_state, finished_parts=None, ): super().__init__(emerge_execution) self.copy_source_range = copy_source_range self.part_number = part_number self.large_file_id = large_file_id self.large_file_upload_state = large_file_upload_state self.finished_parts = finished_parts or {} def execute(self): return self.emerge_execution.services.copy_manager.copy_part( self.large_file_id, self.copy_source_range, self.part_number, self.large_file_upload_state, finished_parts=self.finished_parts, destination_encryption=self.emerge_execution.encryption, source_encryption=self.copy_source_range.encryption, ) class UploadFileExecutionStep(BaseExecutionStep): def __init__(self, emerge_execution, stream_opener, stream_length=None, stream_sha1=None): super().__init__(emerge_execution) self.stream_opener = stream_opener self.stream_length = stream_length self.stream_sha1 = stream_sha1 def execute(self): upload_source = UploadSourceStream( self.stream_opener, stream_length=self.stream_length, stream_sha1=self.stream_sha1, ) execution = self.emerge_execution return execution.services.upload_manager.upload_file( execution.bucket_id, upload_source, execution.file_name, execution.content_type or execution.DEFAULT_CONTENT_TYPE, execution.file_info or {}, execution.progress_listener, encryption=execution.encryption, file_retention=execution.file_retention, legal_hold=execution.legal_hold, custom_upload_timestamp=execution.custom_upload_timestamp, ) class UploadPartExecutionStep(BaseExecutionStep): def __init__( self, emerge_execution, stream_opener, part_number, large_file_id, large_file_upload_state, stream_length=None, stream_sha1=None, finished_parts=None, ): super().__init__(emerge_execution) self.stream_opener = stream_opener self.stream_length = stream_length self.stream_sha1 = stream_sha1 self.part_number = part_number self.large_file_id = large_file_id self.large_file_upload_state = large_file_upload_state self.finished_parts = finished_parts or {} def execute(self): execution = self.emerge_execution upload_source = UploadSourceStream( self.stream_opener, stream_length=self.stream_length, stream_sha1=self.stream_sha1, ) return execution.services.upload_manager.upload_part( execution.bucket_id, self.large_file_id, upload_source, self.part_number, self.large_file_upload_state, finished_parts=self.finished_parts, encryption=execution.encryption, ) b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/planner/000077500000000000000000000000001474454370000236415ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/planner/__init__.py000066400000000000000000000005411474454370000257520ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/emerge/planner/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/planner/part_definition.py000066400000000000000000000120201474454370000273640ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/emerge/planner/part_definition.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from abc import ABCMeta, abstractmethod from functools import partial from typing import TYPE_CHECKING from b2sdk._internal.stream.chained import ChainedStream from b2sdk._internal.stream.range import wrap_with_range from b2sdk._internal.utils import hex_sha1_of_unlimited_stream if TYPE_CHECKING: from b2sdk._internal.transfer.emerge.unbound_write_intent import UnboundSourceBytes class BaseEmergePartDefinition(metaclass=ABCMeta): @abstractmethod def get_length(self): pass @abstractmethod def get_part_id(self): pass @abstractmethod def get_execution_step(self, execution_step_factory): pass def is_hashable(self): return False def get_sha1(self): return None class UploadEmergePartDefinition(BaseEmergePartDefinition): def __init__(self, upload_source: UnboundSourceBytes, relative_offset, length): self.upload_source = upload_source self.relative_offset = relative_offset self.length = length self._sha1 = None def __repr__(self): return ( f'<{self.__class__.__name__} upload_source={repr(self.upload_source)} relative_offset={self.relative_offset} ' f'length={self.length}>' ) def get_length(self): return self.length def get_part_id(self): return self.get_sha1() def is_hashable(self): return True def get_sha1(self): if self._sha1 is None: if self.relative_offset == 0 and self.length == self.upload_source.get_content_length(): # this is part is equal to whole upload source - so we use `get_content_sha1()` # and if sha1 is already given, we skip computing it again self._sha1 = self.upload_source.get_content_sha1() else: with self._get_stream() as stream: self._sha1, _ = hex_sha1_of_unlimited_stream(stream) return self._sha1 def get_execution_step(self, execution_step_factory): return execution_step_factory.create_upload_execution_step( self._get_stream, stream_length=self.length, stream_sha1=self.get_sha1(), ) def _get_stream(self): fp = self.upload_source.open() return wrap_with_range( fp, self.upload_source.get_content_length(), self.relative_offset, self.length ) class UploadSubpartsEmergePartDefinition(BaseEmergePartDefinition): def __init__(self, upload_subparts): self.upload_subparts = upload_subparts self._is_hashable = all(subpart.is_hashable() for subpart in upload_subparts) self._sha1 = None def __repr__(self): return f'<{self.__class__.__name__} upload_subparts={repr(self.upload_subparts)}>' def get_length(self): return sum(subpart.length for subpart in self.upload_subparts) def get_part_id(self): if self.is_hashable(): return self.get_sha1() else: return tuple(subpart.get_subpart_id() for subpart in self.upload_subparts) def is_hashable(self): return self._is_hashable def get_sha1(self): if self._sha1 is None and self.is_hashable(): with self._get_stream() as stream: self._sha1, _ = hex_sha1_of_unlimited_stream(stream) return self._sha1 def get_execution_step(self, execution_step_factory): return execution_step_factory.create_upload_execution_step( partial(self._get_stream, emerge_execution=execution_step_factory.emerge_execution), stream_length=self.get_length(), stream_sha1=self.get_sha1(), ) def _get_stream(self, emerge_execution=None): return ChainedStream( [ subpart.get_stream_opener(emerge_execution=emerge_execution) for subpart in self.upload_subparts ] ) class CopyEmergePartDefinition(BaseEmergePartDefinition): def __init__(self, copy_source, relative_offset, length): self.copy_source = copy_source self.relative_offset = relative_offset self.length = length def __repr__(self): return ( f'<{self.__class__.__name__} copy_source={repr(self.copy_source)} relative_offset={self.relative_offset} ' f'length={self.length}>' ) def get_length(self): return self.length def get_part_id(self): return (self.copy_source.file_id, self.relative_offset, self.length) def get_execution_step(self, execution_step_factory): return execution_step_factory.create_copy_execution_step( self.copy_source.get_copy_source_range(self.relative_offset, self.length) ) b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/planner/planner.py000066400000000000000000000777641474454370000256770ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/emerge/planner/planner.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import hashlib import json import typing from abc import ABCMeta, abstractmethod from collections import deque from math import ceil from b2sdk._internal.exception import InvalidUserInput from b2sdk._internal.http_constants import ( DEFAULT_MAX_PART_SIZE, DEFAULT_MIN_PART_SIZE, DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE, ) from b2sdk._internal.transfer.emerge.planner.part_definition import ( CopyEmergePartDefinition, UploadEmergePartDefinition, UploadSubpartsEmergePartDefinition, ) from b2sdk._internal.transfer.emerge.planner.upload_subpart import ( LocalSourceUploadSubpart, RemoteSourceUploadSubpart, ) from b2sdk._internal.utils import iterator_peek if typing.TYPE_CHECKING: from b2sdk._internal.account_info.abstract import AbstractAccountInfo class UploadBuffer: """data container used by EmergePlanner for temporary storage of write intents""" def __init__(self, start_offset, buff=None): self._start_offset = start_offset self._buff = buff or [] if self._buff: self._end_offset = self._buff[-1][1] else: self._end_offset = self._start_offset @property def start_offset(self): return self._start_offset @property def end_offset(self): return self._end_offset @property def length(self): return self.end_offset - self.start_offset def intent_count(self): return len(self._buff) def get_intent(self, index): return self.get_item(index)[0] def get_item(self, index): return self._buff[index] def iter_items(self): return iter(self._buff) def append(self, intent, fragment_end): self._buff.append((intent, fragment_end)) self._end_offset = fragment_end def get_slice(self, start_idx=None, end_idx=None, start_offset=None): start_idx = start_idx or 0 buff_slice = self._buff[start_idx:end_idx] if start_offset is None: if start_idx == 0: start_offset = self.start_offset else: start_offset = self._buff[start_idx - 1 : start_idx][0][1] return self.__class__(start_offset, buff_slice) def _filter_out_none(*args): return (arg for arg in args if arg is not None) class EmergePlanner: """Creates a list of actions required for advanced creation of an object in the cloud from an iterator of write intent objects""" def __init__( self, min_part_size: int | None = None, recommended_upload_part_size: int | None = None, max_part_size: int | None = None, ): # ensure default values do not break min<=recommended<=max condition, # while respecting user input and not auto fixing if something was provided explicitly self.min_part_size = ( min( DEFAULT_MIN_PART_SIZE, *_filter_out_none(recommended_upload_part_size, max_part_size), ) if min_part_size is None else min_part_size ) self.recommended_upload_part_size = recommended_upload_part_size or max( DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE, self.min_part_size ) self.max_part_size = max_part_size or max( DEFAULT_MAX_PART_SIZE, self.recommended_upload_part_size ) if self.min_part_size > self.recommended_upload_part_size: raise InvalidUserInput( f'min_part_size value ({self.min_part_size}) exceeding recommended_upload_part_size value ({self.recommended_upload_part_size})' ) if self.recommended_upload_part_size > self.max_part_size: raise InvalidUserInput( f'recommended_upload_part_size value ({self.recommended_upload_part_size}) exceeding max_part_size value ({self.max_part_size})' ) @classmethod def from_account_info( cls, account_info: AbstractAccountInfo, min_part_size=None, recommended_upload_part_size=None, max_part_size=None, ): if recommended_upload_part_size is None: recommended_upload_part_size = account_info.get_recommended_part_size() # AccountInfo defaults should not break the min<=recommended<=max condition when # other params were provided explicitly if min_part_size is not None: recommended_upload_part_size = max(recommended_upload_part_size, min_part_size) if max_part_size is not None: recommended_upload_part_size = min(recommended_upload_part_size, max_part_size) kwargs = { 'min_part_size': min_part_size, 'recommended_upload_part_size': recommended_upload_part_size, 'max_part_size': max_part_size, } return cls(**{key: value for key, value in kwargs.items() if value is not None}) def get_emerge_plan(self, write_intents): write_intents = sorted(write_intents, key=lambda intent: intent.destination_offset) # the upload part size recommended by the server causes errors with files larger than 1TB # (with the current 100MB part size and 10000 part count limit). # Therefore here we increase the recommended upload part size if needed. # the constant is for handling mixed upload/copy in concatenate etc max_destination_offset = max(intent.destination_end_offset for intent in write_intents) self.recommended_upload_part_size = max( self.recommended_upload_part_size, min( ceil(1.5 * max_destination_offset / 10000), self.max_part_size, ), ) assert self.min_part_size <= self.recommended_upload_part_size <= self.max_part_size, ( self.min_part_size, self.recommended_upload_part_size, self.max_part_size, ) return self._get_emerge_plan(write_intents, EmergePlan) def get_streaming_emerge_plan(self, write_intent_iterator): return self._get_emerge_plan(write_intent_iterator, StreamingEmergePlan) def get_unbound_emerge_plan(self, write_intent_iterator): """ For unbound streams we skip the whole process of bunching different parts together, validating them and splitting by operation type. We can do this, because: 1. there will be no copy operations at all; 2. we don't want to pull more data than actually needed; 3. all the data is ordered; 4. we don't want anything else to touch our buffers. Furthermore, we're using StreamingEmergePlan, as it checks whether we have one or more chunks to work with, and picks a proper upload method. """ return StreamingEmergePlan(self._get_simple_emerge_parts(write_intent_iterator)) def _get_simple_emerge_parts(self, write_intent_iterator): # Assumption here is that we need to do no magic. We are receiving # a read-only stream that cannot be seeked and is only for uploading # purposes. Moreover, we assume that each write intent we received is # a nice, enclosed buffer with enough data to make the cloud happy. for write_intent in write_intent_iterator: yield UploadEmergePartDefinition( write_intent.outbound_source, relative_offset=0, length=write_intent.length, ) def _get_emerge_plan(self, write_intent_iterator, plan_class): return plan_class( self._get_emerge_parts( self._select_intent_fragments(self._validatation_iterator(write_intent_iterator)) ) ) def _get_emerge_parts(self, intent_fragments_iterator): # This is where the magic happens. Instead of describing typical inputs and outputs here, # We've put them in tests. It is recommended to read those tests before trying to comprehend # the implementation details of this function. min_part_size = self.min_part_size # this stores current intent that we need to process - we may get # it in fragments so we want to glue just by updating `current_end` current_intent = None current_end = 0 upload_buffer = UploadBuffer(0) first = True last = False for intent, fragment_end in intent_fragments_iterator: if current_intent is None: # this is a first loop run - just initialize current intent current_intent = intent current_end = fragment_end continue if intent is current_intent: # new intent is the same as previously processed intent, so lets glue them together # this happens when the caller splits N overlapping intents into overlapping fragments # and two fragments from the same intent end up streaming into here one after the other current_end = fragment_end continue if intent is None: last = True # incoming intent is different - this means that now we have to decide what to do: # if this is a copy intent and we want to copy it server-side, then we have to # flush the whole upload buffer we accumulated so far, but OTOH we may decide that we just want to # append it to upload buffer (see complete, untrivial logic below) and then maybe # flush some upload parts from upload buffer (if there is enough in the buffer) current_len = current_end - upload_buffer.end_offset # should we flush the upload buffer or do we have to add a chunk of the copy first? if current_intent.is_copy() and current_len >= min_part_size: # check if we can flush upload buffer or there is some missing bytes to fill it to `min_part_size` if upload_buffer.intent_count() > 0 and upload_buffer.length < min_part_size: missing_length = min_part_size - upload_buffer.length else: missing_length = 0 if missing_length > 0 and current_len - missing_length < min_part_size: # current intent is *not* a "small copy", but upload buffer is small # and current intent is too short with the buffer to reach the minimum part size # so we append current intent to upload buffer upload_buffer.append(current_intent, current_end) else: if missing_length > 0: # we "borrow" a fragment of current intent to upload buffer # to fill it to minimum part size upload_buffer.append( current_intent, upload_buffer.end_offset + missing_length ) # completely flush the upload buffer for upload_buffer_part in self._buff_split(upload_buffer): yield self._get_upload_part(upload_buffer_part) # split current intent (copy source) to parts and yield copy_parts = self._get_copy_parts( current_intent, start_offset=upload_buffer.end_offset, end_offset=current_end, ) for part in copy_parts: yield part upload_buffer = UploadBuffer(current_end) else: if current_intent.is_copy() and first and last: # this is a single intent "small copy" - we force use of `copy_file` copy_parts = self._get_copy_parts( current_intent, start_offset=upload_buffer.end_offset, end_offset=current_end, ) for part in copy_parts: yield part else: # this is a upload source or "small copy" source (that is *not* single intent) # either way we just add them to upload buffer upload_buffer.append(current_intent, current_end) upload_buffer_parts = list(self._buff_split(upload_buffer)) # we flush all parts excluding last one - we may want to extend # this last part with "incoming" intent in next loop run for upload_buffer_part in upload_buffer_parts[:-1]: yield self._get_upload_part(upload_buffer_part) upload_buffer = upload_buffer_parts[-1] current_intent = intent first = False current_end = fragment_end if current_intent is None: # this is a sentinel - there would be no more fragments - we have to flush upload buffer for upload_buffer_part in self._buff_split(upload_buffer): yield self._get_upload_part(upload_buffer_part) def _get_upload_part(self, upload_buffer): """Build emerge part from upload buffer.""" if upload_buffer.intent_count() == 1 and upload_buffer.get_intent(0).is_upload(): intent = upload_buffer.get_intent(0) relative_offset = upload_buffer.start_offset - intent.destination_offset length = upload_buffer.length definition = UploadEmergePartDefinition(intent.outbound_source, relative_offset, length) else: subparts = [] fragment_start = upload_buffer.start_offset for intent, fragment_end in upload_buffer.iter_items(): relative_offset = fragment_start - intent.destination_offset length = fragment_end - fragment_start if intent.is_upload(): subpart_class = LocalSourceUploadSubpart elif intent.is_copy(): subpart_class = RemoteSourceUploadSubpart else: raise RuntimeError('This cannot happen!!!') subparts.append(subpart_class(intent.outbound_source, relative_offset, length)) fragment_start = fragment_end definition = UploadSubpartsEmergePartDefinition(subparts) return EmergePart(definition) def _get_copy_parts(self, copy_intent, start_offset, end_offset): """Split copy intent to emerge parts.""" fragment_length = end_offset - start_offset part_count = int(fragment_length / self.max_part_size) last_part_length = fragment_length % self.max_part_size if last_part_length == 0: last_part_length = self.max_part_size else: part_count += 1 if part_count == 1: part_sizes = [last_part_length] else: if last_part_length < int(fragment_length / (part_count + 1)): part_count += 1 base_part_size = int(fragment_length / part_count) size_remainder = fragment_length % part_count part_sizes = [ base_part_size + (1 if i < size_remainder else 0) for i in range(part_count) ] copy_source = copy_intent.outbound_source relative_offset = start_offset - copy_intent.destination_offset for part_size in part_sizes: yield EmergePart(CopyEmergePartDefinition(copy_source, relative_offset, part_size)) relative_offset += part_size def _buff_split(self, upload_buffer): """Split upload buffer to parts candidates - smaller upload buffers. :rtype iterator[b2sdk._internal.transfer.emerge.planner.planner.UploadBuffer]: """ if upload_buffer.intent_count() == 0: return tail_buffer = upload_buffer while True: if tail_buffer.length < self.recommended_upload_part_size + self.min_part_size: # `EmergePlanner_buff_partition` can split in such way that tail part # can be smaller than `min_part_size` - to avoid unnecessary download of possible # incoming copy intent, we don't split further yield tail_buffer return head_buff, tail_buffer = self._buff_partition(tail_buffer) yield head_buff def _buff_partition(self, upload_buffer): """Split upload buffer to two parts (smaller upload buffers). In result left part cannot be split more, and nothing can be assumed about right part. :rtype tuple(b2sdk._internal.transfer.emerge.planner.planner.UploadBuffer, b2sdk._internal.transfer.emerge.planner.planner.UploadBuffer): """ left_buff = UploadBuffer(upload_buffer.start_offset) buff_start = upload_buffer.start_offset for idx, (intent, fragment_end) in enumerate(upload_buffer.iter_items()): candidate_size = fragment_end - buff_start if candidate_size > self.recommended_upload_part_size: right_fragment_size = candidate_size - self.recommended_upload_part_size left_buff.append(intent, fragment_end - right_fragment_size) return left_buff, upload_buffer.get_slice( start_idx=idx, start_offset=left_buff.end_offset ) else: left_buff.append(intent, fragment_end) if candidate_size == self.recommended_upload_part_size: return left_buff, upload_buffer.get_slice(start_idx=idx + 1) return left_buff, UploadBuffer(left_buff.end_offset) def _select_intent_fragments(self, write_intent_iterator): """Select overlapping write intent fragments to use. To solve overlapping intents selection, intents can be split to smaller fragments. Those fragments are yielded as soon as decision can be made to use them, so there is possibility that one intent is yielded in multiple fragments. Those would be merged again by higher level iterator that produces emerge parts, but in principle this merging can happen here. Not merging it is a code design decision to make this function easier to implement and also it would allow yielding emerge parts a bit quicker. """ # `protected_intent_length` for upload state is 0, so it would generate at most single intent fragment # every loop iteration, but algorithm is not assuming that - one may one day choose to # protect upload fragments length too - eg. to avoid situation when file is opened to # read just small number of bytes and then switch to another overlapping upload source upload_intents_state = IntentsState() copy_intents_state = IntentsState(protected_intent_length=self.min_part_size) last_sent_offset = 0 incoming_offset = None while True: incoming_intent = next(write_intent_iterator, None) if incoming_intent is None: incoming_offset = None else: incoming_offset = incoming_intent.destination_offset upload_intents = list( upload_intents_state.state_update(last_sent_offset, incoming_offset) ) copy_intents = list(copy_intents_state.state_update(last_sent_offset, incoming_offset)) intent_fragments = self._merge_intent_fragments( last_sent_offset, upload_intents, copy_intents, ) for intent, intent_fragment_end in intent_fragments: yield intent, intent_fragment_end last_sent_offset = intent_fragment_end if incoming_offset is not None and last_sent_offset < incoming_offset: raise ValueError( 'Cannot emerge file with holes. ' f'Found hole range: ({last_sent_offset}, {incoming_offset})' ) if incoming_intent is None: yield ( None, None, ) # lets yield sentinel for cleaner `_get_emerge_parts` implementation return if incoming_intent.is_upload(): upload_intents_state.add(incoming_intent) elif incoming_intent.is_copy(): copy_intents_state.add(incoming_intent) else: raise RuntimeError('This should not happen at all!') def _merge_intent_fragments(self, start_offset, upload_intents, copy_intents): """Select "competing" upload and copy fragments. Upload and copy fragments may overlap so we need to choose right one to use - copy fragments are prioritized unless this fragment is unprotected (we use "protection" as an abstract for "short copy" fragments - meaning upload fragments have higher priority than "short copy") """ upload_intents = deque(upload_intents) copy_intents = deque(copy_intents) while True: upload_intent = copy_intent = None if upload_intents: upload_intent, upload_end, _ = upload_intents[0] if copy_intents: copy_intent, copy_end, copy_protected = copy_intents[0] if upload_intent is not None and copy_intent is not None: if not copy_protected: yield_intent = upload_intent else: yield_intent = copy_intent start_offset = min(upload_end, copy_end) yield yield_intent, start_offset if start_offset >= upload_end: upload_intents.popleft() if start_offset >= copy_end: copy_intents.popleft() elif upload_intent is not None: yield upload_intent, upload_end upload_intents.popleft() elif copy_intent is not None: yield copy_intent, copy_end copy_intents.popleft() else: return def _validatation_iterator(self, write_intents): """Iterate over write intents and validate length and order.""" last_offset = 0 for write_intent in write_intents: if write_intent.length is None: raise ValueError('Planner cannot support write intents of unknown length') if write_intent.destination_offset < last_offset: raise ValueError('Write intent stream have to be sorted by destination offset') last_offset = write_intent.destination_offset yield write_intent class IntentsState: """Store and process state of incoming write intents to solve overlapping intents selection in streaming manner. It does not check if intents are of the same kind (upload/copy), but the intention was to use it to split incoming intents by kind (two intents state are required then). If there would be no need for differentiating incoming intents, this would still work - so intent kind is ignored at this level. To address "short copy" prioritization problem (and avoidance) - ``protected_intent_length`` param was introduced to prevent logic from allowing too small fragments (if it is possible) """ def __init__(self, protected_intent_length=0): self.protected_intent_length = protected_intent_length self._current_intent = None self._next_intent = None self._last_sent_offset = 0 self._incoming_offset = None self._current_intent_start = None self._current_intent_end = None self._next_intent_end = None def add(self, incoming_intent): """Add incoming intent to state. It has to called *after* ``IntentsState.state_update`` but it is not verified. """ if self._next_intent is None: self._set_next_intent(incoming_intent) elif incoming_intent.destination_end_offset > self._next_intent_end: # here either incoming intent starts at the same position as next intent # (and current intent is None in such case - it was just cleared in `state_update` # or it was cleared some time ago - in previous iteratios) or we are in situation # when current and next intent overlaps, and `last_sent_offset` is now set to # incoming intent `destination_offset` - in both cases we want to choose # intent which has larger `destination_end_offset` self._set_next_intent(incoming_intent) def state_update(self, last_sent_offset, incoming_offset): """Update the state using incoming intent offset. It has to be called *before* ``IntentsState.add`` and even if incoming intent would not be added to this intents state. It would yield a state of this stream of intents (like copy or upload) from ``last_sent_offset`` to ``incoming_offset``. So here happens the first stage of solving overlapping intents selection - but write intent iterator can be split to multiple substreams (like copy and upload) so additional stage is required to cover this. """ if self._current_intent is not None: if last_sent_offset >= self._current_intent_end: self._set_current_intent(None, None) # `effective_incoming_offset` is a safeguard after intent iterator is drained if incoming_offset is not None: effective_incoming_offset = incoming_offset elif self._next_intent is not None: effective_incoming_offset = self._next_intent_end elif self._current_intent is not None: effective_incoming_offset = self._current_intent_end else: # intent iterator is drained and this state is empty return if ( self._current_intent is None and self._next_intent is not None and ( self._next_intent.destination_offset != effective_incoming_offset or incoming_offset is None ) ): self._set_current_intent(self._next_intent, last_sent_offset) self._set_next_intent(None) # current and next can be both not None at this point only if they overlap if ( self._current_intent is not None and self._next_intent is not None and effective_incoming_offset > self._current_intent_end ): # incoming intent does not overlap with current intent # so we switch to next because we are sure that we will have to use it anyway # (of course other overriding (eg. "copy" over "upload") state can have # overlapping intent but we have no information about it here) # but we also need to protect current intent length if not self._is_current_intent_protected(): # we were unable to protect current intent, so we can safely rotate self._set_current_intent(self._next_intent, last_sent_offset) self._set_next_intent(None) else: remaining_len = self.protected_intent_length - ( last_sent_offset - self._current_intent_start ) if remaining_len > 0: last_sent_offset += remaining_len if not self._can_be_protected(last_sent_offset, self._next_intent_end): last_sent_offset = self._current_intent_end yield self._current_intent, last_sent_offset, True self._set_current_intent(self._next_intent, last_sent_offset) self._set_next_intent(None) if self._current_intent is not None: yield ( self._current_intent, min(effective_incoming_offset, self._current_intent_end), self._is_current_intent_protected(), ) def _set_current_intent(self, intent, start_offset): self._current_intent = intent if self._current_intent is not None: self._current_intent_end = self._current_intent.destination_end_offset else: self._current_intent_end = None assert start_offset is None self._current_intent_start = start_offset def _set_next_intent(self, intent): self._next_intent = intent if self._next_intent is not None: self._next_intent_end = self._next_intent.destination_end_offset else: self._next_intent_end = None def _is_current_intent_protected(self): """States if current intent is protected. Intent can be split to smaller fragments, but to choose upload over "small copy" we need to know for fragment if it is a "small copy" or not. In result of solving overlapping intents selection there might be a situation when original intent was not a small copy, but in effect it will be used only partially and in effect it may be a "small copy". Algorithm attempts to avoid using smaller fragments than ``protected_intent_length`` but sometimes it may be impossible. So if this function returns ``False`` it means that used length of this intent is smaller than ``protected_intent_length`` and the algorithm was unable to avoid this. """ return self._can_be_protected(self._current_intent_start, self._current_intent_end) def _can_be_protected(self, start, end): return end - start >= self.protected_intent_length class BaseEmergePlan(metaclass=ABCMeta): def __init__(self, emerge_parts): self.emerge_parts = emerge_parts @abstractmethod def is_large_file(self): pass @abstractmethod def get_total_length(self): pass @abstractmethod def get_plan_id(self): pass def enumerate_emerge_parts(self): return enumerate(self.emerge_parts, 1) class EmergePlan(BaseEmergePlan): def __init__(self, emerge_parts): super().__init__(list(emerge_parts)) self._is_large_file = len(self.emerge_parts) > 1 def is_large_file(self): return self._is_large_file def get_total_length(self): return sum(emerge_part.get_length() for emerge_part in self.emerge_parts) def get_plan_id(self): if all(part.is_hashable() for part in self.emerge_parts): return None json_id = json.dumps([emerge_part.get_part_id() for emerge_part in self.emerge_parts]) return hashlib.sha1(json_id.encode()).hexdigest() class StreamingEmergePlan(BaseEmergePlan): def __init__(self, emerge_parts_iterator): emerge_parts_iterator, self._is_large_file = self._peek_for_large_file( emerge_parts_iterator ) super().__init__(emerge_parts_iterator) def is_large_file(self): return self._is_large_file def get_total_length(self): return None def get_plan_id(self): return None def _peek_for_large_file(self, emerge_parts_iterator): peeked, emerge_parts_iterator = iterator_peek(emerge_parts_iterator, 2) if not peeked: raise ValueError('Empty emerge parts iterator') return emerge_parts_iterator, len(peeked) > 1 class EmergePart: def __init__(self, part_definition, verification_ranges=None): self.part_definition = part_definition self.verification_ranges = verification_ranges def __repr__(self): return f'<{self.__class__.__name__} part_definition={repr(self.part_definition)}>' def get_length(self): return self.part_definition.get_length() def get_execution_step(self, execution_step_factory): return self.part_definition.get_execution_step(execution_step_factory) def get_part_id(self): return self.part_definition.get_part_id() def is_hashable(self): return self.part_definition.is_hashable() def get_sha1(self): return self.part_definition.get_sha1() b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/planner/upload_subpart.py000066400000000000000000000065301474454370000272430ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/emerge/planner/upload_subpart.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import io from abc import ABCMeta, abstractmethod from functools import partial from b2sdk._internal.stream.chained import StreamOpener from b2sdk._internal.stream.range import wrap_with_range from b2sdk._internal.utils import hex_sha1_of_unlimited_stream class BaseUploadSubpart(metaclass=ABCMeta): def __init__(self, outbound_source, relative_offset, length): self.outbound_source = outbound_source self.relative_offset = relative_offset self.length = length def __repr__(self): return ( f'<{self.__class__.__name__} outbound_source={repr(self.outbound_source)} relative_offset={self.relative_offset} ' f'length={self.length}>' ) @abstractmethod def get_subpart_id(self): pass @abstractmethod def get_stream_opener(self, emerge_execution=None): pass def is_hashable(self): return False class RemoteSourceUploadSubpart(BaseUploadSubpart): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._download_buffer_cache = None def get_subpart_id(self): return (self.outbound_source.file_id, self.relative_offset, self.length) def get_stream_opener(self, emerge_execution=None): if emerge_execution is None: raise RuntimeError('Cannot open remote source without emerge execution instance.') return CachedBytesStreamOpener(partial(self._download, emerge_execution)) def _download(self, emerge_execution): url = emerge_execution.services.session.get_download_url_by_id(self.outbound_source.file_id) absolute_offset = self.outbound_source.offset + self.relative_offset range_ = (absolute_offset, absolute_offset + self.length - 1) with io.BytesIO() as bytes_io: downloaded_file = emerge_execution.services.download_manager.download_file_from_url( url, range_=range_, encryption=self.outbound_source.encryption ) downloaded_file.save(bytes_io) return bytes_io.getvalue() class LocalSourceUploadSubpart(BaseUploadSubpart): def get_subpart_id(self): with self._get_stream() as stream: sha1, _ = hex_sha1_of_unlimited_stream(stream) return sha1 def get_stream_opener(self, emerge_execution=None): return self._get_stream def _get_stream(self): fp = self.outbound_source.open() return wrap_with_range( fp, self.outbound_source.get_content_length(), self.relative_offset, self.length ) def is_hashable(self): return True class CachedBytesStreamOpener(StreamOpener): def __init__(self, bytes_data_callback): self.bytes_data_callback = bytes_data_callback self._bytes_data_cache = None def __call__(self): if self._bytes_data_cache is None: self._bytes_data_cache = self.bytes_data_callback() return io.BytesIO(self._bytes_data_cache) def cleanup(self): self._bytes_data_cache = None b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/unbound_write_intent.py000066400000000000000000000204061474454370000270230ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/emerge/unbound_write_intent.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import hashlib import io import queue from typing import Callable, Iterator from b2sdk._internal.transfer.emerge.exception import UnboundStreamBufferTimeout from b2sdk._internal.transfer.emerge.write_intent import WriteIntent from b2sdk._internal.transfer.outbound.upload_source import AbstractUploadSource class IOWrapper(io.BytesIO): """ Wrapper for BytesIO that knows when it has been read in full. Note that this stream should go through ``emerge_unbound``, as it's the only one that skips ``_get_emerge_parts`` and pushes buffers to the cloud exactly as they come. This way we can (somewhat) rely on check whether reading of this wrapper returned no more data. It is assumed that this object is owned by a single thread at a time. For that reason, no additional synchronisation is provided. """ def __init__( self, data: bytes | bytearray, release_function: Callable[[], None], ): """ Prepares a new ``io.BytesIO`` structure that will call a ``release_function`` when buffer is read in full. ``release_function`` can be called from another thread. It is called exactly once, when the read is concluded and the resource is about to be released :param data: data to be provided as a stream :param release_function: function to be called when resource will be released """ super().__init__(data) self.release_function = release_function def close(self): if not self.closed: self.release_function() return super().close() class UnboundSourceBytes(AbstractUploadSource): """ Upload source that deals with a chunk of unbound data. It ensures that the data it provides doesn't have to be iterated over more than once. To do that, we have ensured that both length and sha1 is known. Also, it should be used only with ``emerge_unbound``, as it's the only plan that pushes buffers directly to the cloud. """ def __init__( self, bytes_data: bytearray, release_function: Callable[[], None], ): """ Prepares a new ```UploadSource`` that can be used with ``WriteIntent``. Calculates SHA1 and length of the data. :param bytes_data: data that should be uploaded, IOWrapper for this data is created. :param release_function: function to be called when all the ``bytes_data`` is uploaded. """ self.length = len(bytes_data) # Prepare sha1 of the chunk upfront to ensure that nothing iterates over the stream but the upload. self.chunk_sha1 = hashlib.sha1(bytes_data).hexdigest() self.stream = IOWrapper(bytes_data, release_function) def get_content_sha1(self): return self.chunk_sha1 def open(self): return self.stream def get_content_length(self): return self.length class UnboundWriteIntentGenerator: """ Generator that creates new write intents as data is streamed from an external source. It tries to ensure that at most ``queue_size`` buffers with size ``buffer_size_bytes`` are allocated at any given moment. """ def __init__( self, read_only_source, buffer_size_bytes: int, read_size: int, queue_size: int, queue_timeout_seconds: float, ): """ Prepares a new intent generator for a given source. ``queue_size`` is handled on a best-effort basis. It's possible, in rare cases, that there will be more buffers available at once. With current implementation that would be the case when the whole buffer was read, but on the very last byte the server stopped responding and a retry is issued. :param read_only_source: Python object that has a ``read`` method. :param buffer_size_bytes: Size of a single buffer that we're to download from the source and push to the cloud. :param read_size: Size of a single read to be performed on ``read_only_source``. :param queue_size: Maximal amount of buffers that will be created. :param queue_timeout_seconds: Iterator will wait at most this many seconds for an empty slot for a buffer. After that time it's considered an error. """ assert ( queue_size >= 1 and read_size > 0 and buffer_size_bytes > 0 and queue_timeout_seconds > 0.0 ) self.read_only_source = read_only_source self.read_size = read_size self.buffer_size_bytes = buffer_size_bytes self.buffer_limit_queue = queue.Queue(maxsize=queue_size) self.queue_timeout_seconds = queue_timeout_seconds self.buffer = bytearray() self.leftovers_buffer = bytearray() def iterator(self) -> Iterator[WriteIntent]: """ Creates new ``WriteIntent`` objects as the data is pulled from the ``read_only_source``. """ datastream_done = False offset = 0 while not datastream_done: self._wait_for_free_buffer_slot() # In very small buffer sizes and large read sizes we could # land with multiple buffers read at once. This should happen # only in tests. self._trim_to_leftovers() while len(self.buffer) < self.buffer_size_bytes: data = self.read_only_source.read(self.read_size) if len(data) == 0: datastream_done = True break self.buffer += data self._trim_to_leftovers() # If we've just started a new buffer and got an empty read on it, # we have no data to send and the process is finished. if len(self.buffer) == 0: self._release_buffer() break source = UnboundSourceBytes(self.buffer, self._release_buffer) intent = WriteIntent(source, destination_offset=offset) yield intent offset += len(self.buffer) self._rotate_leftovers() # If we didn't stream anything, we should still provide # at least an empty WriteIntent, so that the file will be created. if offset == 0: source = UnboundSourceBytes(bytearray(), release_function=lambda: None) yield WriteIntent(source, destination_offset=offset) def _trim_to_leftovers(self) -> None: if len(self.buffer) <= self.buffer_size_bytes: return remainder = len(self.buffer) - self.buffer_size_bytes buffer_view = memoryview(self.buffer) self.leftovers_buffer += buffer_view[-remainder:] # This conversion has little to no implication on performance. self.buffer = bytearray(buffer_view[:-remainder]) def _rotate_leftovers(self) -> None: self.buffer = self.leftovers_buffer self.leftovers_buffer = bytearray() def _wait_for_free_buffer_slot(self) -> None: # Inserted item is only a placeholder. If we fail to insert it in given time, it means # that system is unable to process data quickly enough. By default, this timeout is around # a really large value (counted in minutes, not seconds) to indicate weird behaviour. try: self.buffer_limit_queue.put(1, timeout=self.queue_timeout_seconds) except queue.Full: raise UnboundStreamBufferTimeout() def _release_buffer(self) -> None: # Pull one element from the queue of waiting elements. # Note that it doesn't matter which element we pull. # Each of them is just a placeholder. Since we know that we've put them there, # there is no need to actually wait. The queue should contain at least one element if we got here. try: self.buffer_limit_queue.get_nowait() except queue.Empty as error: # pragma: nocover raise RuntimeError('Buffer pulled twice from the queue.') from error b2-sdk-python-2.8.0/b2sdk/_internal/transfer/emerge/write_intent.py000066400000000000000000000062261474454370000252750ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/emerge/write_intent.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal.utils import Sha1HexDigest class WriteIntent: """Wrapper for outbound source that defines destination offset.""" def __init__(self, outbound_source, destination_offset=0): """ :param b2sdk.v2.OutboundTransferSource outbound_source: data source (remote or local) :param int destination_offset: point of start in destination file """ if outbound_source.get_content_length() is None: raise ValueError('Cannot wrap outbound source of unknown length') self.outbound_source = outbound_source self.destination_offset = destination_offset def __repr__(self): return ( f'<{self.__class__.__name__} outbound_source={repr(self.outbound_source)} ' f'destination_offset={self.destination_offset} id={id(self)}>' ) @property def length(self): """Length of the write intent. :rtype: int """ return self.outbound_source.get_content_length() @property def destination_end_offset(self): """Offset of source end in destination file. :rtype: int """ return self.destination_offset + self.length def is_copy(self): """States if outbound source is remote source and requires copying. :rtype: bool """ return self.outbound_source.is_copy() def is_upload(self): """States if outbound source is local source and requires uploading. :rtype: bool """ return self.outbound_source.is_upload() def get_content_sha1(self) -> Sha1HexDigest | None: """ Return a 40-character string containing the hex SHA1 checksum, which can be used as the `large_file_sha1` entry. This method is only used if a large file is constructed from only a single source. If that source's hash is known, the result file's SHA1 checksum will be the same and can be copied. If the source's sha1 is unknown and can't be calculated, `None` is returned. :rtype str: """ return self.outbound_source.get_content_sha1() @classmethod def wrap_sources_iterator(cls, outbound_sources_iterator): """Helper that wraps outbound sources iterator with write intents. Can be used in cases similar to concatenate to automatically compute destination offsets :param: iterator[b2sdk.v2.OutboundTransferSource] outbound_sources_iterator: iterator of outbound sources :rtype: generator[b2sdk.v2.WriteIntent] """ current_position = 0 for outbound_source in outbound_sources_iterator: length = outbound_source.get_content_length() write_intent = WriteIntent(outbound_source, destination_offset=current_position) current_position += length yield write_intent b2-sdk-python-2.8.0/b2sdk/_internal/transfer/inbound/000077500000000000000000000000001474454370000223745ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/transfer/inbound/__init__.py000066400000000000000000000005321474454370000245050ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/inbound/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/transfer/inbound/download_manager.py000066400000000000000000000106531474454370000262540ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/inbound/download_manager.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from b2sdk._internal.encryption.setting import EncryptionSetting from b2sdk._internal.exception import ( InvalidRange, ) from b2sdk._internal.progress import AbstractProgressListener, DoNothingProgressListener from b2sdk._internal.utils import B2TraceMetaAbstract from ...utils.thread_pool import ThreadPoolMixin from ..transfer_manager import TransferManager from .downloaded_file import DownloadedFile from .downloader.parallel import ParallelDownloader from .downloader.simple import SimpleDownloader logger = logging.getLogger(__name__) class DownloadManager(TransferManager, ThreadPoolMixin, metaclass=B2TraceMetaAbstract): """ Handle complex actions around downloads to free raw_api from that responsibility. """ # minimum size of a download chunk DEFAULT_MIN_PART_SIZE = 100 * 1024 * 1024 # block size used when downloading file. If it is set to a high value, # progress reporting will be jumpy, if it's too low, it impacts CPU MIN_CHUNK_SIZE = 8192 # ~1MB file will show ~1% progress increment MAX_CHUNK_SIZE = 1024**2 PARALLEL_DOWNLOADER_CLASS = staticmethod(ParallelDownloader) SIMPLE_DOWNLOADER_CLASS = staticmethod(SimpleDownloader) def __init__( self, write_buffer_size: int | None = None, check_hash: bool = True, max_download_streams_per_file: int | None = None, **kwargs, ): """ Initialize the DownloadManager using the given services object. """ super().__init__(**kwargs) self.strategies = [ self.PARALLEL_DOWNLOADER_CLASS( min_part_size=self.DEFAULT_MIN_PART_SIZE, min_chunk_size=self.MIN_CHUNK_SIZE, max_chunk_size=max(self.MAX_CHUNK_SIZE, write_buffer_size or 0), align_factor=write_buffer_size, thread_pool=self._thread_pool, check_hash=check_hash, max_streams=max_download_streams_per_file, ), self.SIMPLE_DOWNLOADER_CLASS( min_chunk_size=self.MIN_CHUNK_SIZE, max_chunk_size=max(self.MAX_CHUNK_SIZE, write_buffer_size or 0), align_factor=write_buffer_size, thread_pool=self._thread_pool, check_hash=check_hash, ), ] self.write_buffer_size = write_buffer_size self.check_hash = check_hash def download_file_from_url( self, url: str, progress_listener: AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: EncryptionSetting | None = None, ) -> DownloadedFile: """ Download file by URL. :param url: url from which the file should be downloaded :param progress_listener: where to notify about downloading progress :param range_: 2-element tuple containing data of http Range header :param b2sdk.v2.EncryptionSetting encryption: encryption setting (``None`` if unknown) """ progress_listener = progress_listener or DoNothingProgressListener() with self.services.session.download_file_from_url( url, range_=range_, encryption=encryption, ) as response: download_version = self.services.api.download_version_factory.from_response_headers( response.headers ) if range_ is not None: # 2021-05-20: unfortunately for a read of a complete object server does not return the 'Content-Range' header if (range_[1] - range_[0] + 1) != download_version.content_length: raise InvalidRange(download_version.content_length, range_) return DownloadedFile( download_version=download_version, download_manager=self, range_=range_, response=response, encryption=encryption, progress_listener=progress_listener, write_buffer_size=self.write_buffer_size, check_hash=self.check_hash, ) b2-sdk-python-2.8.0/b2sdk/_internal/transfer/inbound/downloaded_file.py000066400000000000000000000235711474454370000260750ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/inbound/downloaded_file.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import contextlib import io import logging import pathlib import sys from typing import TYPE_CHECKING, BinaryIO from requests.models import Response from b2sdk._internal.exception import ( ChecksumMismatch, DestinationDirectoryDoesntAllowOperation, DestinationDirectoryDoesntExist, DestinationError, DestinationIsADirectory, DestinationParentIsNotADirectory, TruncatedOutput, ) from b2sdk._internal.utils import set_file_mtime from b2sdk._internal.utils.filesystem import _IS_WINDOWS, points_to_fifo, points_to_stdout try: from typing_extensions import Literal except ImportError: from typing import Literal from ...encryption.setting import EncryptionSetting from ...file_version import DownloadVersion from ...progress import AbstractProgressListener from ...stream.progress import WritingStreamWithProgress if TYPE_CHECKING: from .download_manager import DownloadManager logger = logging.getLogger(__name__) class MtimeUpdatedFile(io.IOBase): """ Helper class that facilitates updating a files mod_time after closing. Over the time this class has grown, and now it also adds better exception handling. Usage: .. code-block: python downloaded_file = bucket.download_file_by_id('b2_file_id') with MtimeUpdatedFile('some_local_path', mod_time_millis=downloaded_file.download_version.mod_time_millis) as file: downloaded_file.save(file) # 'some_local_path' has the mod_time set according to metadata in B2 """ def __init__( self, path_: str | pathlib.Path, mod_time_millis: int, mode: Literal['wb', 'wb+'] = 'wb+', buffering: int | None = None, ): self.path = pathlib.Path(path_) if isinstance(path_, str) else path_ self.mode = mode self.buffering = buffering if buffering is not None else -1 self.mod_time_to_set = mod_time_millis self.file = None @property def path_(self) -> str: return str(self.path) @path_.setter def path_(self, value: str) -> None: self.path = pathlib.Path(value) def write(self, value): """ This method is overwritten (monkey-patched) in __enter__ for performance reasons """ raise NotImplementedError def read(self, *a): """ This method is overwritten (monkey-patched) in __enter__ for performance reasons """ raise NotImplementedError def seekable(self) -> bool: return self.file.seekable() def seek(self, offset, whence=0): return self.file.seek(offset, whence) def tell(self): return self.file.tell() def __enter__(self): try: path = self.path if not path.parent.exists(): raise DestinationDirectoryDoesntExist() if not path.parent.is_dir(): raise DestinationParentIsNotADirectory() # This ensures consistency on *nix and Windows. Windows doesn't seem to raise ``IsADirectoryError`` at all, # so with this we actually can differentiate between permissions errors and target being a directory. if path.exists() and path.is_dir(): raise DestinationIsADirectory() except PermissionError as ex: raise DestinationDirectoryDoesntAllowOperation() from ex try: self.file = open( self.path, self.mode, buffering=self.buffering, ) except PermissionError as ex: raise DestinationDirectoryDoesntAllowOperation() from ex self.write = self.file.write self.read = self.file.read self.mode = self.file.mode return self def __exit__(self, exc_type, exc_val, exc_tb): self.file.close() set_file_mtime(self.path_, self.mod_time_to_set) def __str__(self): return str(self.path) class DownloadedFile: """ Result of a successful download initialization. Holds information about file's metadata and allows to perform the download. """ def __init__( self, download_version: DownloadVersion, download_manager: DownloadManager, range_: tuple[int, int] | None, response: Response, encryption: EncryptionSetting | None, progress_listener: AbstractProgressListener, write_buffer_size=None, check_hash=True, ): self.download_version = download_version self.download_manager = download_manager self.range_ = range_ self.response = response self.encryption = encryption self.progress_listener = progress_listener self.download_strategy = None self.write_buffer_size = write_buffer_size self.check_hash = check_hash def _validate_download(self, bytes_read, actual_sha1): if ( self.download_version.content_encoding is not None and self.download_version.api.api_config.decode_content ): return if self.range_ is None: if bytes_read != self.download_version.content_length: raise TruncatedOutput(bytes_read, self.download_version.content_length) if ( self.check_hash and self.download_version.content_sha1 != 'none' and actual_sha1 != self.download_version.content_sha1 ): raise ChecksumMismatch( checksum_type='sha1', expected=self.download_version.content_sha1, actual=actual_sha1, ) else: desired_length = self.range_[1] - self.range_[0] + 1 if bytes_read != desired_length: raise TruncatedOutput(bytes_read, desired_length) def save(self, file: BinaryIO, allow_seeking: bool | None = None) -> None: """ Read data from B2 cloud and write it to a file-like object :param file: a file-like object :param allow_seeking: if False, download strategies that rely on seeking to write data (parallel strategies) will be discarded. """ if allow_seeking is None: allow_seeking = file.seekable() elif allow_seeking and not file.seekable(): logger.warning('File is not seekable, disabling strategies that require seeking') allow_seeking = False if allow_seeking: # check if file allows reading from arbitrary position try: file.read(0) except io.UnsupportedOperation: logger.warning( 'File is seekable, but does not allow reads, disabling strategies that require seeking' ) allow_seeking = False if self.progress_listener: file = WritingStreamWithProgress(file, self.progress_listener) if self.range_ is not None: total_bytes = self.range_[1] - self.range_[0] + 1 else: total_bytes = self.download_version.content_length self.progress_listener.set_total_bytes(total_bytes) for strategy in self.download_manager.strategies: if strategy.is_suitable(self.download_version, allow_seeking): break else: raise ValueError('no strategy suitable for download was found!') self.download_strategy = strategy bytes_read, actual_sha1 = strategy.download( file, response=self.response, download_version=self.download_version, session=self.download_manager.services.session, encryption=self.encryption, ) self._validate_download(bytes_read, actual_sha1) def save_to( self, path_: str | pathlib.Path, mode: Literal['wb', 'wb+'] | None = None, allow_seeking: bool | None = None, ) -> None: """ Open a local file and write data from B2 cloud to it, also update the mod_time. :param path_: path to file to be opened :param mode: mode in which the file should be opened :param allow_seeking: if False, download strategies that rely on seeking to write data (parallel strategies) will be discarded. """ path_ = pathlib.Path(path_) is_stdout = points_to_stdout(path_) if is_stdout or points_to_fifo(path_): if mode not in (None, 'wb'): raise DestinationError(f'invalid mode requested {mode!r} for FIFO file {path_!r}') if is_stdout and _IS_WINDOWS: if self.write_buffer_size and self.write_buffer_size not in ( -1, io.DEFAULT_BUFFER_SIZE, ): logger.warning( 'Unable to set arbitrary write_buffer_size for stdout on Windows' ) context = contextlib.nullcontext(sys.stdout.buffer) else: context = open(path_, 'wb', buffering=self.write_buffer_size or -1) try: with context as file: return self.save(file, allow_seeking=allow_seeking) finally: if not is_stdout: set_file_mtime(path_, self.download_version.mod_time_millis) with MtimeUpdatedFile( path_, mod_time_millis=self.download_version.mod_time_millis, mode=mode or 'wb+', buffering=self.write_buffer_size, ) as file: return self.save(file, allow_seeking=allow_seeking) b2-sdk-python-2.8.0/b2sdk/_internal/transfer/inbound/downloader/000077500000000000000000000000001474454370000245325ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/transfer/inbound/downloader/__init__.py000066400000000000000000000005451474454370000266470ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/inbound/downloader/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/transfer/inbound/downloader/abstract.py000066400000000000000000000120751474454370000267140ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/inbound/downloader/abstract.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import hashlib from abc import abstractmethod from concurrent.futures import ThreadPoolExecutor from io import IOBase from requests.models import Response from b2sdk._internal.encryption.setting import EncryptionSetting from b2sdk._internal.file_version import DownloadVersion from b2sdk._internal.session import B2Session from b2sdk._internal.utils import B2TraceMetaAbstract from b2sdk._internal.utils.range_ import Range class EmptyHasher: def __init__(self, *args, **kwargs): pass def update(self, data): pass def digest(self): return b'' def hexdigest(self): return '' def copy(self): return self class AbstractDownloader(metaclass=B2TraceMetaAbstract): """ Abstract class for downloaders. :var REQUIRES_SEEKING: if True, the downloader requires the ability to seek in the file object. :var SUPPORTS_DECODE_CONTENT: if True, the downloader supports decoded HTTP streams. In practice, this means that the downloader can handle HTTP responses which already have the content decoded per Content-Encoding and, more likely than not, of a different length than requested. """ REQUIRES_SEEKING = True SUPPORTS_DECODE_CONTENT = True DEFAULT_THREAD_POOL_CLASS = staticmethod(ThreadPoolExecutor) DEFAULT_ALIGN_FACTOR = 4096 def __init__( self, thread_pool: ThreadPoolExecutor | None = None, force_chunk_size: int | None = None, min_chunk_size: int | None = None, max_chunk_size: int | None = None, align_factor: int | None = None, check_hash: bool = True, **kwargs, ): align_factor = align_factor or self.DEFAULT_ALIGN_FACTOR assert force_chunk_size is not None or ( min_chunk_size is not None and max_chunk_size is not None and 0 < min_chunk_size <= max_chunk_size and max_chunk_size >= align_factor ) self._min_chunk_size = min_chunk_size self._max_chunk_size = max_chunk_size self._forced_chunk_size = force_chunk_size self._align_factor = align_factor self._check_hash = check_hash self._thread_pool = ( thread_pool if thread_pool is not None else self.DEFAULT_THREAD_POOL_CLASS() ) super().__init__(**kwargs) def _get_hasher(self): if self._check_hash: return hashlib.sha1() return EmptyHasher() def _get_chunk_size(self, content_length: int | None): if self._forced_chunk_size is not None: return self._forced_chunk_size ideal = max(content_length // 1000, self._align_factor) non_aligned = min(max(ideal, self._min_chunk_size), self._max_chunk_size) aligned = non_aligned // self._align_factor * self._align_factor return aligned @classmethod def _get_remote_range(cls, response: Response, download_version: DownloadVersion): """ Get a range from response or original request (as appropriate). :param response: requests.Response of initial request :param download_version: b2sdk.v2.DownloadVersion :return: a range object """ if 'Range' in response.request.headers: return Range.from_header(response.request.headers['Range']) return download_version.range_ def is_suitable(self, download_version: DownloadVersion, allow_seeking: bool): """ Analyze download_version (possibly against options passed earlier to constructor to find out whether the given download request should be handled by this downloader). """ if self.REQUIRES_SEEKING and not allow_seeking: return False if ( not self.SUPPORTS_DECODE_CONTENT and download_version.content_encoding and download_version.api.api_config.decode_content ): return False return True @abstractmethod def download( self, file: IOBase, response: Response, download_version: DownloadVersion, session: B2Session, encryption: EncryptionSetting | None = None, ) -> tuple[int, str]: """ Download target to a file-like object. :param file: file-like object to write to :param response: requests.Response of b2_download_url_by_* endpoint with the target object :param download_version: DownloadVersion of an object being downloaded :param session: B2Session to be used for downloading :param encryption: optional Encryption setting :return: (bytes_read, actual_sha1) please note bytes_read may be different from bytes written to a file object if decode_content=True """ pass b2-sdk-python-2.8.0/b2sdk/_internal/transfer/inbound/downloader/parallel.py000066400000000000000000000544221474454370000267070ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/inbound/downloader/parallel.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging import platform import queue import threading from concurrent import futures from io import IOBase from time import perf_counter_ns from requests import RequestException from requests.models import Response from b2sdk._internal.encryption.setting import EncryptionSetting from b2sdk._internal.exception import B2Error, TruncatedOutput from b2sdk._internal.file_version import DownloadVersion from b2sdk._internal.session import B2Session from b2sdk._internal.utils.range_ import Range from .abstract import AbstractDownloader from .stats_collector import StatsCollector logger = logging.getLogger(__name__) class ParallelDownloader(AbstractDownloader): """ Downloader using threads to download&write multiple parts of an object in parallel. Each part is downloaded by its own thread, while all writes are done by additional dedicated thread. This can increase performance even for a small file, as fetching & writing can be done in parallel. """ # situations to consider: # # local file start local file end # | | # | | # | write range start write range end | # | | | | # v v v v # ####################################################################### # | | | | | | # \ / \ / \ / \ / \ / # part 1 part 2 part 3 part 4 part 5 # / \ / \ / \ / \ / \ # | | | | | | # ####################################################################### # ^ ^ # | | # cloud file start cloud file end # FINISH_HASHING_BUFFER_SIZE = 1024**2 SUPPORTS_DECODE_CONTENT = False def __init__(self, min_part_size: int, max_streams: int | None = None, **kwargs): """ :param max_streams: maximum number of simultaneous streams :param min_part_size: minimum amount of data a single stream will retrieve, in bytes """ super().__init__(**kwargs) self.max_streams = max_streams self.min_part_size = min_part_size def _get_number_of_streams(self, content_length: int) -> int: num_streams = content_length // self.min_part_size if self.max_streams is not None: num_streams = min(num_streams, self.max_streams) else: max_threadpool_workers = getattr(self._thread_pool, '_max_workers', None) if max_threadpool_workers is not None: num_streams = min(num_streams, max_threadpool_workers) return max(num_streams, 1) def download( self, file: IOBase, response: Response, download_version: DownloadVersion, session: B2Session, encryption: EncryptionSetting | None = None, ): """ Download a file from given url using parallel download sessions and stores it in the given download_destination. """ remote_range = self._get_remote_range(response, download_version) hasher = self._get_hasher() if remote_range.size() == 0: response.close() return 0, hasher.hexdigest() actual_size = remote_range.size() start_file_position = file.tell() parts_to_download = list( gen_parts( remote_range, Range(start_file_position, start_file_position + actual_size - 1), part_count=self._get_number_of_streams(download_version.content_length), ) ) first_part = parts_to_download[0] with WriterThread(file, max_queue_depth=len(parts_to_download) * 2) as writer: self._get_parts( response, session, writer, hasher, first_part, parts_to_download[1:], self._get_chunk_size(actual_size), encryption=encryption, ) bytes_written = writer.total # At this point the hasher already consumed the data until the end of first stream. # Consume the rest of the file to complete the hashing process if self._check_hash: # we skip hashing if we would not check it - hasher object is actually a EmptyHasher instance # but we avoid here reading whole file (except for the first part) from disk again before_hash = perf_counter_ns() self._finish_hashing(first_part, file, hasher, download_version.content_length) after_hash = perf_counter_ns() logger.info( 'download stats | %s | %s total: %.3f ms', file, 'finish_hash', (after_hash - before_hash) / 1000000, ) return bytes_written, hasher.hexdigest() def _finish_hashing(self, first_part, file, hasher, content_length): end_of_first_part = first_part.local_range.end + 1 file.seek(end_of_first_part) file_read = file.read last_offset = first_part.local_range.start + content_length current_offset = end_of_first_part stop = False while 1: data = file_read(self.FINISH_HASHING_BUFFER_SIZE) if not data: break if current_offset + len(data) >= last_offset: to_hash = data[: last_offset - current_offset] stop = True else: to_hash = data hasher.update(data) current_offset += len(to_hash) if stop: break def _get_parts( self, response, session, writer, hasher, first_part, parts_to_download, chunk_size, encryption, ): stream = self._thread_pool.submit( download_first_part, response, hasher, session, writer, first_part, chunk_size, encryption=encryption, ) streams = {stream} for part in parts_to_download: stream = self._thread_pool.submit( download_non_first_part, response.request.url, session, writer, part, chunk_size, encryption=encryption, ) streams.add(stream) # free-up resources & check for early failures try: streams_futures = futures.wait( streams, timeout=0, return_when=futures.FIRST_COMPLETED ) except futures.TimeoutError: pass else: try: for stream in streams_futures.done: stream.result() except Exception: if platform.python_implementation() == 'PyPy': # Await all threads to avoid PyPy hanging bug. # https://github.com/pypy/pypy/issues/4994#issuecomment-2258962665 futures.wait(streams_futures.not_done) raise streams = streams_futures.not_done futures.wait(streams) for stream in streams: stream.result() class WriterThread(threading.Thread): """ A thread responsible for keeping a queue of data chunks to write to a file-like object and for actually writing them down. Since a single thread is responsible for synchronization of the writes, we avoid a lot of issues between userspace and kernelspace that would normally require flushing buffers between the switches of the writer. That would kill performance and not synchronizing would cause data corruption (probably we'd end up with a file with unexpected blocks of zeros preceding the range of the writer that comes second and writes further into the file). The object of this class is also responsible for backpressure: if items are added to the queue faster than they can be written (see GCP VMs with standard PD storage with faster CPU and network than local storage, https://github.com/Backblaze/B2_Command_Line_Tool/issues/595), then ``obj.queue.put(item)`` will block, slowing down the producer. The recommended minimum value of ``max_queue_depth`` is equal to the amount of producer threads, so that if all producers submit a part at the exact same time (right after network issue, for example, or just after starting the read), they can continue their work without blocking. The writer should be able to store at least one data chunk before a new one is retrieved, but it is not guaranteed. Therefore, the recommended value of ``max_queue_depth`` is higher - a double of the amount of producers, so that spikes on either end (many producers submit at the same time / consumer has a latency spike) can be accommodated without sacrificing performance. Please note that a size of the chunk and the queue depth impact the memory footprint. In a default setting as of writing this, that might be 10 downloads, 8 producers, 1MB buffers, 2 buffers each = 8*2*10 = 160 MB (+ python buffers, operating system etc). """ def __init__(self, file, max_queue_depth): self.file = file self.queue = queue.Queue(max_queue_depth) self.total = 0 self.stats_collector = StatsCollector(str(self.file), 'writer', 'seek') super().__init__() def run(self): file = self.file queue_get = self.queue.get stats_collector_read = self.stats_collector.read stats_collector_other = self.stats_collector.other stats_collector_write = self.stats_collector.write with self.stats_collector.total: while 1: with stats_collector_read: shutdown, offset, data = queue_get() if shutdown: break with stats_collector_other: file.seek(offset) with stats_collector_write: file.write(data) self.total += len(data) def __enter__(self): self.start() return self def queue_write(self, offset: int, data: bytes) -> None: self.queue.put((False, offset, data)) def __exit__(self, exc_type, exc_val, exc_tb): self.queue.put((True, None, None)) self.join() self.stats_collector.report() def download_first_part( response: Response, hasher, session: B2Session, writer: WriterThread, part_to_download: PartToDownload, chunk_size: int, encryption: EncryptionSetting | None = None, ) -> None: """ :param response: response of the original GET call :param hasher: hasher object to feed to as the stream is written :param session: B2 API session :param writer: thread responsible for writing downloaded data :param part_to_download: definition of the part to be downloaded :param chunk_size: size (in bytes) of read data chunks :param encryption: encryption mode, algorithm and key """ # This function contains a loop that has heavy impact on performance. # It has not been broken down to several small functions due to fear of # performance overhead of calling a python function. Advanced performance optimization # techniques are in use here, for example avoiding internal python getattr calls by # caching function signatures in local variables. Most of this code was written in # times where python 2.7 (or maybe even 2.6) had to be supported, so maybe some # of those optimizations could be removed without affecting performance. # # Due to reports of hard to debug performance issues, this code has also been riddled # with performance measurements. A known issue is GCP VMs which have more network speed # than storage speed, but end users have different issues with network and storage. # Basic tools to figure out where the time is being spent is a must for long-term # maintainability. writer_queue_put = writer.queue_write hasher_update = hasher.update local_range_start = part_to_download.local_range.start actual_part_size = part_to_download.local_range.size() starting_cloud_range = part_to_download.cloud_range bytes_read = 0 url = response.request.url max_attempts = 15 # this is hardcoded because we are going to replace the entire retry interface soon, so we'll avoid deprecation here and keep it private attempt = 0 stats_collector = StatsCollector( response.url, f'{local_range_start}:{part_to_download.local_range.end}', 'hash' ) stats_collector_read = stats_collector.read stats_collector_other = stats_collector.other stats_collector_write = stats_collector.write with stats_collector.total: attempt += 1 logger.debug( 'download part %s %s attempt: %i, bytes read already: %i', url, part_to_download, attempt, bytes_read, ) response_iterator = response.iter_content(chunk_size=chunk_size) part_not_completed = True while part_not_completed: with stats_collector_read: try: data = next(response_iterator) except StopIteration: break except RequestException: if attempt < max_attempts: break else: raise predicted_bytes_read = bytes_read + len(data) if predicted_bytes_read > actual_part_size: to_write = data[: actual_part_size - bytes_read] part_not_completed = False else: to_write = data with stats_collector_write: writer_queue_put(local_range_start + bytes_read, to_write) with stats_collector_other: hasher_update(to_write) bytes_read += len(to_write) # since we got everything we need from original response, close the socket and free the buffer # to avoid a timeout exception during hashing and other trouble response.close() while attempt < max_attempts and bytes_read < actual_part_size: attempt += 1 cloud_range = starting_cloud_range.subrange(bytes_read, actual_part_size - 1) logger.debug( 'download part %s %s attempt: %i, bytes read already: %i. Getting range %s now.', url, part_to_download, attempt, bytes_read, cloud_range, ) try: with session.download_file_from_url( url, cloud_range.as_tuple(), encryption=encryption, ) as response: response_iterator = response.iter_content(chunk_size=chunk_size) while True: with stats_collector_read: try: to_write = next(response_iterator) except StopIteration: break with stats_collector_write: writer_queue_put(local_range_start + bytes_read, to_write) with stats_collector_other: hasher_update(to_write) bytes_read += len(to_write) except (B2Error, RequestException) as e: should_retry = e.should_retry_http() if isinstance(e, B2Error) else True if should_retry and attempt < max_attempts: logger.debug( 'Download of %s %s attempt %d failed with %s, retrying', url, part_to_download, attempt, e, ) else: raise stats_collector.report() if bytes_read != actual_part_size: logger.error( 'Failed to download %s %s; Downloaded %d/%d after %d attempts', url, part_to_download, bytes_read, actual_part_size, attempt, ) raise TruncatedOutput( bytes_read=bytes_read, file_size=actual_part_size, ) else: logger.debug( 'Successfully downloaded %s %s; Downloaded %d/%d after %d attempts', url, part_to_download, bytes_read, actual_part_size, attempt, ) def download_non_first_part( url: str, session: B2Session, writer: WriterThread, part_to_download: PartToDownload, chunk_size: int, encryption: EncryptionSetting | None = None, ) -> None: """ :param url: download URL :param session: B2 API session :param writer: thread responsible for writing downloaded data :param part_to_download: definition of the part to be downloaded :param chunk_size: size (in bytes) of read data chunks :param encryption: encryption mode, algorithm and key """ writer_queue_put = writer.queue_write local_range_start = part_to_download.local_range.start actual_part_size = part_to_download.local_range.size() starting_cloud_range = part_to_download.cloud_range bytes_read = 0 max_attempts = 15 # this is hardcoded because we are going to replace the entire retry interface soon, so we'll avoid deprecation here and keep it private attempt = 0 stats_collector = StatsCollector( url, f'{local_range_start}:{part_to_download.local_range.end}', 'none' ) stats_collector_read = stats_collector.read stats_collector_write = stats_collector.write while attempt < max_attempts and bytes_read < actual_part_size: attempt += 1 cloud_range = starting_cloud_range.subrange(bytes_read, actual_part_size - 1) logger.debug( 'download part %s %s attempt: %i, bytes read already: %i. Getting range %s now.', url, part_to_download, attempt, bytes_read, cloud_range, ) with stats_collector.total: try: with session.download_file_from_url( url, cloud_range.as_tuple(), encryption=encryption, ) as response: response_iterator = response.iter_content(chunk_size=chunk_size) while True: with stats_collector_read: try: to_write = next(response_iterator) except StopIteration: break with stats_collector_write: writer_queue_put(local_range_start + bytes_read, to_write) bytes_read += len(to_write) except (B2Error, RequestException) as e: should_retry = e.should_retry_http() if isinstance(e, B2Error) else True if should_retry and attempt < max_attempts: logger.debug( 'Download of %s %s attempt %d failed with %s, retrying', url, part_to_download, attempt, e, ) else: raise stats_collector.report() if bytes_read != actual_part_size: logger.error( 'Failed to download %s %s; Downloaded %d/%d after %d attempts', url, part_to_download, bytes_read, actual_part_size, attempt, ) raise TruncatedOutput( bytes_read=bytes_read, file_size=actual_part_size, ) else: logger.debug( 'Successfully downloaded %s %s; Downloaded %d/%d after %d attempts', url, part_to_download, bytes_read, actual_part_size, attempt, ) class PartToDownload: """ Hold the range of a file to download, and the range of the local file where it should be stored. """ def __init__(self, cloud_range, local_range): self.cloud_range = cloud_range self.local_range = local_range def __repr__(self): return f'PartToDownload({self.cloud_range}, {self.local_range})' def gen_parts(cloud_range, local_range, part_count): """ Generate a sequence of PartToDownload to download a large file as a collection of parts. """ # We don't support DECODE_CONTENT here, hence cloud&local ranges have to be the same assert cloud_range.size() == local_range.size(), (cloud_range.size(), local_range.size()) assert 0 < part_count <= cloud_range.size() offset = 0 remaining_size = cloud_range.size() for i in range(part_count): # This rounds down, so if the parts aren't all the same size, # the smaller parts will come first. this_part_size = remaining_size // (part_count - i) part = PartToDownload( cloud_range.subrange(offset, offset + this_part_size - 1), local_range.subrange(offset, offset + this_part_size - 1), ) logger.debug('created part to download: %s', part) yield part offset += this_part_size remaining_size -= this_part_size b2-sdk-python-2.8.0/b2sdk/_internal/transfer/inbound/downloader/simple.py000066400000000000000000000074721474454370000264070ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/inbound/downloader/simple.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from io import IOBase from requests.models import Response from b2sdk._internal.encryption.setting import EncryptionSetting from b2sdk._internal.file_version import DownloadVersion from b2sdk._internal.session import B2Session from .abstract import AbstractDownloader logger = logging.getLogger(__name__) class SimpleDownloader(AbstractDownloader): REQUIRES_SEEKING = False SUPPORTS_DECODE_CONTENT = True def _download( self, file: IOBase, response: Response, download_version: DownloadVersion, session: B2Session, encryption: EncryptionSetting | None = None, ): digest = self._get_hasher() actual_size = self._get_remote_range(response, download_version).size() if actual_size == 0: response.close() return 0, digest.hexdigest() chunk_size = self._get_chunk_size(actual_size) decoded_bytes_read = 0 for data in response.iter_content(chunk_size=chunk_size): file.write(data) digest.update(data) decoded_bytes_read += len(data) bytes_read = response.raw.tell() response.close() assert ( actual_size >= 1 ) # code below does `actual_size - 1`, but it should never reach that part with an empty file # now, normally bytes_read == download_version.content_length, but sometimes there is a timeout # or something and the server closes connection, while neither tcp or http have a problem # with the truncated output, so we detect it here and try to continue num_tries = 5 # this is hardcoded because we are going to replace the entire retry interface soon, so we'll avoid deprecation here and keep it private retries_left = num_tries - 1 while retries_left and bytes_read < download_version.content_length: new_range = self._get_remote_range( response, download_version, ).subrange(bytes_read, actual_size - 1) # original response is not closed at this point yet, as another layer is responsible for closing it, so a new socket might be allocated, # but this is a very rare case and so it is not worth the optimization logger.debug( 're-download attempts remaining: %i, bytes read: %i (decoded: %i). Getting range %s now.', retries_left, bytes_read, decoded_bytes_read, new_range, ) with session.download_file_from_url( response.request.url, new_range.as_tuple(), encryption=encryption, ) as followup_response: for data in followup_response.iter_content( chunk_size=self._get_chunk_size(actual_size) ): file.write(data) digest.update(data) decoded_bytes_read += len(data) bytes_read += followup_response.raw.tell() retries_left -= 1 return bytes_read, digest.hexdigest() def download( self, file: IOBase, response: Response, download_version: DownloadVersion, session: B2Session, encryption: EncryptionSetting | None = None, ): future = self._thread_pool.submit( self._download, file, response, download_version, session, encryption ) return future.result() b2-sdk-python-2.8.0/b2sdk/_internal/transfer/inbound/downloader/stats_collector.py000066400000000000000000000060611474454370000303130ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/inbound/downloader/stats_collector.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from dataclasses import ( dataclass, field, ) from time import perf_counter_ns from typing import ( Any, ) logger = logging.getLogger(__name__) class SingleStatsCollector: TO_MS = 1_000_000 def __init__(self): self.latest_entry: int | None = None self.sum_of_all_entries: int = 0 self.started_perf_timer: int | None = None def __enter__(self) -> None: self.started_perf_timer = perf_counter_ns() def __exit__(self, exc_type: type, exc_val: Exception, exc_tb: Any) -> None: time_diff = perf_counter_ns() - self.started_perf_timer self.latest_entry = time_diff self.sum_of_all_entries += time_diff self.started_perf_timer = None @property def sum_ms(self) -> float: return self.sum_of_all_entries / self.TO_MS @property def latest_ms(self) -> float: return self.latest_entry / self.TO_MS @property def has_any_entry(self) -> bool: return self.latest_entry is not None @dataclass class StatsCollector: name: str #: file name or object url detail: str #: description of the thread, ex. "10000000:20000000" or "writer" other_name: str #: other statistic, typically "seek" or "hash" total: SingleStatsCollector = field(default_factory=SingleStatsCollector) other: SingleStatsCollector = field(default_factory=SingleStatsCollector) write: SingleStatsCollector = field(default_factory=SingleStatsCollector) read: SingleStatsCollector = field(default_factory=SingleStatsCollector) def report(self): if self.read.has_any_entry: logger.info('download stats | %s | TTFB: %.3f ms', self, self.read.latest_ms) logger.info( 'download stats | %s | read() without TTFB: %.3f ms', self, (self.read.sum_of_all_entries - self.read.latest_entry) / self.read.TO_MS, ) if self.other.has_any_entry: logger.info( 'download stats | %s | %s total: %.3f ms', self, self.other_name, self.other.sum_ms ) if self.write.has_any_entry: logger.info('download stats | %s | write() total: %.3f ms', self, self.write.sum_ms) if self.total.has_any_entry: basic_operation_time = ( self.write.sum_of_all_entries + self.other.sum_of_all_entries + self.read.sum_of_all_entries ) overhead = self.total.sum_of_all_entries - basic_operation_time logger.info( 'download stats | %s | overhead: %.3f ms', self, overhead / self.total.TO_MS ) def __str__(self): return f'{self.name}[{self.detail}]' b2-sdk-python-2.8.0/b2sdk/_internal/transfer/outbound/000077500000000000000000000000001474454370000225755ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/transfer/outbound/__init__.py000066400000000000000000000005331474454370000247070ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/outbound/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/_internal/transfer/outbound/copy_manager.py000066400000000000000000000242221474454370000256150ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/outbound/copy_manager.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from b2sdk._internal.encryption.setting import EncryptionMode, EncryptionSetting from b2sdk._internal.exception import AlreadyFailed, CopyArgumentsMismatch, SSECKeyIdMismatchInCopy from b2sdk._internal.file_lock import FileRetentionSetting, LegalHold from b2sdk._internal.http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME from b2sdk._internal.progress import AbstractProgressListener from b2sdk._internal.raw_api import MetadataDirectiveMode from b2sdk._internal.transfer.transfer_manager import TransferManager from b2sdk._internal.utils.thread_pool import ThreadPoolMixin logger = logging.getLogger(__name__) class CopyManager(TransferManager, ThreadPoolMixin): """ Handle complex actions around server side copy to free raw_api from that responsibility. """ MAX_LARGE_FILE_SIZE = 10 * 1000 * 1000 * 1000 * 1000 # 10 TB @property def account_info(self): return self.services.session.account_info def copy_file( self, copy_source, file_name, content_type, file_info, destination_bucket_id, progress_listener, destination_encryption: EncryptionSetting | None = None, source_encryption: EncryptionSetting | None = None, legal_hold: LegalHold | None = None, file_retention: FileRetentionSetting | None = None, ): # Run small copies in the same thread pool as large file copies, # so that they share resources during a sync. return self._thread_pool.submit( self._copy_small_file, copy_source, file_name, content_type=content_type, file_info=file_info, destination_bucket_id=destination_bucket_id, progress_listener=progress_listener, destination_encryption=destination_encryption, source_encryption=source_encryption, legal_hold=legal_hold, file_retention=file_retention, ) def copy_part( self, large_file_id, part_copy_source, part_number, large_file_upload_state, finished_parts=None, destination_encryption: EncryptionSetting | None = None, source_encryption: EncryptionSetting | None = None, ): return self._thread_pool.submit( self._copy_part, large_file_id, part_copy_source, part_number, large_file_upload_state, finished_parts=finished_parts, destination_encryption=destination_encryption, source_encryption=source_encryption, ) def _copy_part( self, large_file_id, part_copy_source, part_number, large_file_upload_state, finished_parts, destination_encryption: EncryptionSetting | None, source_encryption: EncryptionSetting | None, ): """ Copy a file part to started large file. :param :param str bucket_id: a bucket ID :param large_file_id: a large file ID :param b2sdk.v2.CopySource part_copy_source: copy source that represents a range (not necessarily a whole file) :param b2sdk.v2.LargeFileUploadState large_file_upload_state: state object for progress reporting on large file upload :param dict,None finished_parts: dictionary of known finished parts, keys are part numbers, values are instances of :class:`~b2sdk.v2.Part` :param b2sdk.v2.EncryptionSetting destination_encryption: encryption settings for the destination (``None`` if unknown) :param b2sdk.v2.EncryptionSetting source_encryption: encryption settings for the source (``None`` if unknown) """ # b2_copy_part doesn't need SSE-B2. Large file encryption is decided on b2_start_large_file. if ( destination_encryption is not None and destination_encryption.mode == EncryptionMode.SSE_B2 ): destination_encryption = None # Check if this part was uploaded before if finished_parts is not None and part_number in finished_parts: # Report this part finished part = finished_parts[part_number] large_file_upload_state.update_part_bytes(part.content_length) # Return SHA1 hash return {'contentSha1': part.content_sha1} # if another part has already had an error there's no point in # uploading this part if large_file_upload_state.has_error(): raise AlreadyFailed(large_file_upload_state.get_error_message()) response = self.services.session.copy_part( part_copy_source.file_id, large_file_id, part_number, bytes_range=part_copy_source.get_bytes_range(), destination_server_side_encryption=destination_encryption, source_server_side_encryption=source_encryption, ) large_file_upload_state.update_part_bytes(response['contentLength']) return response def _copy_small_file( self, copy_source, file_name, content_type, file_info, destination_bucket_id, progress_listener: AbstractProgressListener, destination_encryption: EncryptionSetting | None, source_encryption: EncryptionSetting | None, legal_hold: LegalHold | None = None, file_retention: FileRetentionSetting | None = None, ): progress_listener.set_total_bytes(copy_source.get_content_length() or 0) bytes_range = copy_source.get_bytes_range() if content_type is None: if file_info is not None: raise CopyArgumentsMismatch('File info can be set only when content type is set') metadata_directive = MetadataDirectiveMode.COPY else: if file_info is None: raise CopyArgumentsMismatch( 'File info can be not set only when content type is not set' ) metadata_directive = MetadataDirectiveMode.REPLACE metadata_directive, file_info, content_type = self.establish_sse_c_file_metadata( metadata_directive=metadata_directive, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=destination_encryption, source_server_side_encryption=source_encryption, source_file_info=copy_source.source_file_info, source_content_type=copy_source.source_content_type, ) response = self.services.session.copy_file( copy_source.file_id, file_name, bytes_range=bytes_range, metadata_directive=metadata_directive, content_type=content_type, file_info=file_info, destination_bucket_id=destination_bucket_id, destination_server_side_encryption=destination_encryption, source_server_side_encryption=source_encryption, legal_hold=legal_hold, file_retention=file_retention, ) file_version = self.services.api.file_version_factory.from_api_response(response) progress_listener.bytes_completed(file_version.size) return file_version @classmethod def establish_sse_c_file_metadata( cls, metadata_directive: MetadataDirectiveMode, destination_file_info: dict | None, destination_content_type: str | None, destination_server_side_encryption: EncryptionSetting | None, source_server_side_encryption: EncryptionSetting | None, source_file_info: dict | None, source_content_type: str | None, ): assert metadata_directive in (MetadataDirectiveMode.REPLACE, MetadataDirectiveMode.COPY) if metadata_directive == MetadataDirectiveMode.REPLACE: if destination_server_side_encryption: destination_file_info = destination_server_side_encryption.add_key_id_to_file_info( destination_file_info ) return metadata_directive, destination_file_info, destination_content_type source_key_id = None destination_key_id = None if ( destination_server_side_encryption is not None and destination_server_side_encryption.key is not None and destination_server_side_encryption.key.key_id is not None ): destination_key_id = destination_server_side_encryption.key.key_id if ( source_server_side_encryption is not None and source_server_side_encryption.key is not None and source_server_side_encryption.key.key_id is not None ): source_key_id = source_server_side_encryption.key.key_id if source_key_id == destination_key_id: return metadata_directive, destination_file_info, destination_content_type if source_file_info is None or source_content_type is None: raise SSECKeyIdMismatchInCopy( f'attempting to copy file using {MetadataDirectiveMode.COPY} without providing source_file_info ' f'and source_content_type for differing sse_c_key_ids: source="{source_key_id}", ' f'destination="{destination_key_id}"' ) destination_file_info = source_file_info.copy() destination_file_info.pop(SSE_C_KEY_ID_FILE_INFO_KEY_NAME, None) if destination_server_side_encryption: destination_file_info = destination_server_side_encryption.add_key_id_to_file_info( destination_file_info ) destination_content_type = source_content_type return MetadataDirectiveMode.REPLACE, destination_file_info, destination_content_type b2-sdk-python-2.8.0/b2sdk/_internal/transfer/outbound/copy_source.py000066400000000000000000000054211474454370000255030ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/outbound/copy_source.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal.encryption.setting import EncryptionSetting from b2sdk._internal.http_constants import LARGE_FILE_SHA1 from b2sdk._internal.transfer.outbound.outbound_source import OutboundTransferSource class CopySource(OutboundTransferSource): def __init__( self, file_id, offset=0, length=None, encryption: EncryptionSetting | None = None, source_file_info=None, source_content_type=None, ): if not length and offset > 0: raise ValueError('Cannot copy with non zero offset and unknown or zero length') self.file_id = file_id self.length = length self.offset = offset self.encryption = encryption self.source_file_info = source_file_info self.source_content_type = source_content_type def __repr__(self): return ( f'<{self.__class__.__name__} file_id={self.file_id} offset={self.offset} length={self.length} id={id(self)}, encryption={self.encryption},' f'source_content_type={self.source_content_type}>, source_file_info={self.source_file_info}' ) def get_content_length(self): return self.length def is_upload(self): return False def is_copy(self): return True def get_bytes_range(self): if not self.length: if self.offset > 0: # auto mode should get file info and create correct copy source (with length) raise ValueError( 'cannot return bytes range for non zero offset and unknown or zero length' ) return None return self.offset, self.offset + self.length - 1 def get_copy_source_range(self, relative_offset, range_length): if self.length is not None and range_length + relative_offset > self.length: raise ValueError('Range length overflow source length') range_offset = self.offset + relative_offset return self.__class__( self.file_id, range_offset, range_length, encryption=self.encryption, source_file_info=self.source_file_info, source_content_type=self.source_content_type, ) def get_content_sha1(self): if self.offset or self.length: # this is a copy of only a range of the source, can't copy the SHA1 return None return self.source_file_info.get(LARGE_FILE_SHA1) b2-sdk-python-2.8.0/b2sdk/_internal/transfer/outbound/large_file_upload_state.py000066400000000000000000000041021474454370000300010ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/outbound/large_file_upload_state.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import threading class LargeFileUploadState: """ Track the status of uploading a large file, accepting updates from the tasks that upload each of the parts. The aggregated progress is passed on to a ProgressListener that reports the progress for the file as a whole. This class is THREAD SAFE. """ def __init__(self, file_progress_listener): """ :param b2sdk.v2.AbstractProgressListener file_progress_listener: a progress listener object to use. Use :py:class:`b2sdk.v2.DoNothingProgressListener` to disable. """ self.lock = threading.RLock() self.error_message = None self.file_progress_listener = file_progress_listener self.part_number_to_part_state = {} self.bytes_completed = 0 def set_error(self, message): """ Set an error message. :param str message: an error message """ with self.lock: self.error_message = message def has_error(self): """ Check whether an error occurred. :rtype: bool """ with self.lock: return self.error_message is not None def get_error_message(self): """ Fetche an error message. :return: an error message :rtype: str """ with self.lock: assert self.has_error() return self.error_message def update_part_bytes(self, bytes_delta): """ Update listener progress info. :param int bytes_delta: number of bytes to increase a progress for """ with self.lock: self.bytes_completed += bytes_delta self.file_progress_listener.bytes_completed(self.bytes_completed) b2-sdk-python-2.8.0/b2sdk/_internal/transfer/outbound/outbound_source.py000066400000000000000000000034131474454370000263670ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/outbound/outbound_source.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from abc import ABCMeta, abstractmethod from b2sdk._internal.utils import Sha1HexDigest class OutboundTransferSource(metaclass=ABCMeta): """Abstract class for defining outbound transfer sources. Supported outbound transfer sources are: * :class:`b2sdk.v2.CopySource` * :class:`b2sdk.v2.UploadSourceBytes` * :class:`b2sdk.v2.UploadSourceLocalFile` * :class:`b2sdk.v2.UploadSourceLocalFileRange` * :class:`b2sdk.v2.UploadSourceStream` * :class:`b2sdk.v2.UploadSourceStreamRange` """ @abstractmethod def get_content_length(self) -> int: """ Returns the number of bytes of data in the file. """ @abstractmethod def get_content_sha1(self) -> Sha1HexDigest | None: """ Return a 40-character string containing the hex SHA1 checksum, which can be used as the `large_file_sha1` entry. This method is only used if a large file is constructed from only a single source. If that source's hash is known, the result file's SHA1 checksum will be the same and can be copied. If the source's sha1 is unknown and can't be calculated, `None` is returned. """ @abstractmethod def is_upload(self) -> bool: """ Returns True if outbound source is an upload source. """ @abstractmethod def is_copy(self) -> bool: """ Returns True if outbound source is a copy source. """ b2-sdk-python-2.8.0/b2sdk/_internal/transfer/outbound/progress_reporter.py000066400000000000000000000031311474454370000267330ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/outbound/progress_reporter.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal.progress import AbstractProgressListener class PartProgressReporter(AbstractProgressListener): """ An adapter that listens to the progress of upload a part and gives the information to a :py:class:`b2sdk._internal.transfer.outbound.large_file_upload_state.LargeFileUploadState`. Accepts absolute bytes_completed from the uploader, and reports deltas to the :py:class:`b2sdk._internal.transfer.outbound.large_file_upload_state.LargeFileUploadState`. The bytes_completed for the part will drop back to 0 on a retry, which will result in a negative delta. """ def __init__(self, large_file_upload_state, *args, **kwargs): """ :param b2sdk._internal.transfer.outbound.large_file_upload_state.LargeFileUploadState large_file_upload_state: object to relay the progress to """ super().__init__(*args, **kwargs) self.large_file_upload_state = large_file_upload_state self.prev_byte_count = 0 def bytes_completed(self, byte_count): self.large_file_upload_state.update_part_bytes(byte_count - self.prev_byte_count) self.prev_byte_count = byte_count def close(self): pass def set_total_bytes(self, total_byte_count): pass b2-sdk-python-2.8.0/b2sdk/_internal/transfer/outbound/upload_manager.py000066400000000000000000000226061474454370000261330ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/outbound/upload_manager.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from contextlib import ExitStack from typing import TYPE_CHECKING, TypeVar from b2sdk._internal.encryption.setting import EncryptionMode, EncryptionSetting from b2sdk._internal.exception import ( AlreadyFailed, B2Error, MaxRetriesExceeded, ) from b2sdk._internal.file_lock import FileRetentionSetting, LegalHold from b2sdk._internal.http_constants import HEX_DIGITS_AT_END from b2sdk._internal.stream.hashing import StreamWithHash from b2sdk._internal.stream.progress import ReadingStreamWithProgress from ...utils.thread_pool import ThreadPoolMixin from ..transfer_manager import TransferManager from .progress_reporter import PartProgressReporter logger = logging.getLogger(__name__) if TYPE_CHECKING: from b2sdk._internal.transfer.outbound.upload_source import AbstractUploadSource _TypeUploadSource = TypeVar('_TypeUploadSource', bound=AbstractUploadSource) class UploadManager(TransferManager, ThreadPoolMixin): """ Handle complex actions around uploads to free raw_api from that responsibility. """ MAX_UPLOAD_ATTEMPTS = 5 @property def account_info(self): return self.services.session.account_info def upload_file( self, bucket_id, upload_source, file_name, content_type, file_info, progress_listener, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): f = self._thread_pool.submit( self._upload_small_file, bucket_id, upload_source, file_name, content_type, file_info, progress_listener, encryption, file_retention, legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) return f def upload_part( self, bucket_id, file_id, part_upload_source: _TypeUploadSource, part_number, large_file_upload_state, finished_parts=None, encryption: EncryptionSetting = None, ): f = self._thread_pool.submit( self._upload_part, bucket_id, file_id, part_upload_source, part_number, large_file_upload_state, finished_parts, encryption, ) return f def _upload_part( self, bucket_id, file_id, part_upload_source: _TypeUploadSource, part_number, large_file_upload_state, finished_parts, encryption: EncryptionSetting, ): """ Upload a file part to started large file. :param :param str bucket_id: a bucket ID :param file_id: a large file ID :param b2sdk.v2.AbstractUploadSource part_upload_source: upload source that reads only required range :param b2sdk.v2.LargeFileUploadState large_file_upload_state: state object for progress reporting on large file upload :param dict,None finished_parts: dictionary of known finished parts, keys are part numbers, values are instances of :class:`~b2sdk.v2.Part` :param b2sdk.v2.EncryptionSetting encryption: encryption setting (``None`` if unknown) """ # b2_upload_part doesn't need SSE-B2. Large file encryption is decided on b2_start_large_file. if encryption is not None and encryption.mode == EncryptionMode.SSE_B2: encryption = None # Check if this part was uploaded before if finished_parts is not None and part_number in finished_parts: # Report this part finished part = finished_parts[part_number] large_file_upload_state.update_part_bytes(part_upload_source.get_content_length()) # Return SHA1 hash return {'contentSha1': part.content_sha1} # Set up a progress listener part_progress_listener = PartProgressReporter(large_file_upload_state) # Retry the upload as needed exception_list = [] with ExitStack() as stream_guard: part_stream = None def close_stream_callback(stream): if not stream.closed: stream.close() for _ in range(self.MAX_UPLOAD_ATTEMPTS): # if another part has already had an error there's no point in # uploading this part if large_file_upload_state.has_error(): raise AlreadyFailed(large_file_upload_state.get_error_message()) try: # reuse the stream in case of retry part_stream = part_stream or part_upload_source.open() # register stream closing callback only when reading is finally concluded stream_guard.callback(close_stream_callback, part_stream) content_length = part_upload_source.get_content_length() input_stream = ReadingStreamWithProgress( part_stream, part_progress_listener, length=content_length ) if part_upload_source.is_sha1_known(): content_sha1 = part_upload_source.get_content_sha1() else: input_stream = StreamWithHash(input_stream, stream_length=content_length) content_sha1 = HEX_DIGITS_AT_END # it is important that `len()` works on `input_stream` response = self.services.session.upload_part( file_id, part_number, len(input_stream), content_sha1, input_stream, server_side_encryption=encryption, # todo: client side encryption ) if content_sha1 == HEX_DIGITS_AT_END: content_sha1 = input_stream.hash assert content_sha1 == response['contentSha1'] return response except B2Error as e: if not e.should_retry_upload(): raise exception_list.append(e) self.account_info.clear_bucket_upload_data(bucket_id) large_file_upload_state.set_error(str(exception_list[-1])) raise MaxRetriesExceeded(self.MAX_UPLOAD_ATTEMPTS, exception_list) def _upload_small_file( self, bucket_id, upload_source, file_name, content_type, file_info, progress_listener, encryption: EncryptionSetting | None = None, file_retention: FileRetentionSetting | None = None, legal_hold: LegalHold | None = None, custom_upload_timestamp: int | None = None, ): content_length = upload_source.get_content_length() exception_info_list = [] progress_listener.set_total_bytes(content_length) for _ in range(self.MAX_UPLOAD_ATTEMPTS): try: with upload_source.open() as file: input_stream = ReadingStreamWithProgress( file, progress_listener, length=content_length ) if upload_source.is_sha1_known(): content_sha1 = upload_source.get_content_sha1() else: input_stream = StreamWithHash(input_stream, stream_length=content_length) content_sha1 = HEX_DIGITS_AT_END # it is important that `len()` works on `input_stream` response = self.services.session.upload_file( bucket_id, file_name, len(input_stream), content_type, content_sha1, file_info, input_stream, server_side_encryption=encryption, # todo: client side encryption file_retention=file_retention, legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) if content_sha1 == HEX_DIGITS_AT_END: content_sha1 = input_stream.hash assert ( content_sha1 == 'do_not_verify' or content_sha1 == response['contentSha1'] ), '{} != {}'.format(content_sha1, response['contentSha1']) return self.services.api.file_version_factory.from_api_response(response) except B2Error as e: if not e.should_retry_upload(): raise exception_info_list.append(e) self.account_info.clear_bucket_upload_data(bucket_id) raise MaxRetriesExceeded(self.MAX_UPLOAD_ATTEMPTS, exception_info_list) b2-sdk-python-2.8.0/b2sdk/_internal/transfer/outbound/upload_source.py000066400000000000000000000303701474454370000260160ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/outbound/upload_source.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import hashlib import io import logging import os from abc import abstractmethod from enum import Enum, auto, unique from typing import Callable from b2sdk._internal.exception import InvalidUploadSource from b2sdk._internal.file_version import BaseFileVersion from b2sdk._internal.http_constants import DEFAULT_MIN_PART_SIZE from b2sdk._internal.stream.range import RangeOfInputStream, wrap_with_range from b2sdk._internal.transfer.outbound.copy_source import CopySource from b2sdk._internal.transfer.outbound.outbound_source import OutboundTransferSource from b2sdk._internal.utils import ( IncrementalHexDigester, Sha1HexDigest, hex_sha1_of_stream, hex_sha1_of_unlimited_stream, ) logger = logging.getLogger(__name__) @unique class UploadMode(Enum): """Mode of file uploads""" FULL = auto() #: always upload the whole file INCREMENTAL = auto() #: use incremental uploads when possible class AbstractUploadSource(OutboundTransferSource): """ The source of data for uploading to b2. """ @abstractmethod def get_content_sha1(self) -> Sha1HexDigest | None: """ Returns a 40-character string containing the hex SHA1 checksum of the data in the file. """ @abstractmethod def open(self) -> io.IOBase: """ Returns a binary file-like object from which the data can be read. """ def is_upload(self) -> bool: return True def is_copy(self) -> bool: return False def is_sha1_known(self) -> bool: """ Returns information whether SHA1 of the source is currently available. Note that negative result doesn't mean that SHA1 is not available. Calling ``get_content_sha1`` can still provide a valid digest. """ return False class UploadSourceBytes(AbstractUploadSource): def __init__( self, data_bytes: bytes | bytearray, content_sha1: Sha1HexDigest | None = None, ): """ Initialize upload source using given bytes. :param data_bytes: Data that is to be uploaded. :param content_sha1: SHA1 hexdigest of the data, or ``None``. """ self.data_bytes = data_bytes self.content_sha1 = content_sha1 def __repr__(self) -> str: return '<{classname} data={data} id={id}>'.format( classname=self.__class__.__name__, data=str(self.data_bytes[:20]) + '...' if len(self.data_bytes) > 20 else self.data_bytes, id=id(self), ) def get_content_length(self) -> int: return len(self.data_bytes) def get_content_sha1(self) -> Sha1HexDigest | None: if self.content_sha1 is None: self.content_sha1 = hashlib.sha1(self.data_bytes).hexdigest() return self.content_sha1 def open(self): return io.BytesIO(self.data_bytes) def is_sha1_known(self) -> bool: return self.content_sha1 is not None class UploadSourceLocalFileBase(AbstractUploadSource): def __init__( self, local_path: os.PathLike | str, content_sha1: Sha1HexDigest | None = None, ): """ Initialize upload source using provided path. :param local_path: Any path-like object that points to a file to be uploaded. :param content_sha1: SHA1 hexdigest of the data, or ``None``. """ self.local_path = local_path self.content_length = 0 self.content_sha1 = content_sha1 self.check_path_and_get_size() def check_path_and_get_size(self) -> None: if not os.path.isfile(self.local_path): raise InvalidUploadSource(self.local_path) self.content_length = os.path.getsize(self.local_path) def __repr__(self) -> str: return ( f'<{self.__class__.__name__} local_path={self.local_path} content_length={self.content_length} ' f'content_sha1={self.content_sha1} id={id(self)}>' ) def get_content_length(self) -> int: return self.content_length def get_content_sha1(self) -> Sha1HexDigest | None: if self.content_sha1 is None: self.content_sha1 = self._hex_sha1_of_file() return self.content_sha1 def open(self): return open(self.local_path, 'rb') def _hex_sha1_of_file(self) -> Sha1HexDigest: with self.open() as f: return hex_sha1_of_stream(f, self.content_length) def is_sha1_known(self) -> bool: return self.content_sha1 is not None class UploadSourceLocalFileRange(UploadSourceLocalFileBase): def __init__( self, local_path: os.PathLike | str, content_sha1: Sha1HexDigest | None = None, offset: int = 0, length: int | None = None, ): """ Initialize upload source using provided path. :param local_path: Any path-like object that points to a file to be uploaded. :param content_sha1: SHA1 hexdigest of the data, or ``None``. :param offset: Position in the file where upload should start from. :param length: Amount of data to be uploaded. If ``None``, length of the remainder of the file is taken. """ super().__init__(local_path, content_sha1) self.file_size = self.content_length self.offset = offset if length is None: self.content_length = self.file_size - self.offset else: if length + self.offset > self.file_size: raise ValueError('Range length overflow file size') self.content_length = length def __repr__(self) -> str: return ( f'<{self.__class__.__name__} local_path={self.local_path} offset={self.offset} ' f'content_length={self.content_length} content_sha1={self.content_sha1} id={id(self)}>' ) def open(self): fp = super().open() return wrap_with_range(fp, self.file_size, self.offset, self.content_length) class UploadSourceLocalFile(UploadSourceLocalFileBase): def get_incremental_sources( self, file_version: BaseFileVersion, min_part_size: int | None = None, ) -> list[OutboundTransferSource]: """ Split the upload into a copy and upload source constructing an incremental upload This will return a list of upload sources. If the upload cannot split, the method will return [self]. """ if not file_version: logger.debug( 'Fallback to full upload for %s -- no matching file on server', self.local_path ) return [self] min_part_size = min_part_size or DEFAULT_MIN_PART_SIZE if file_version.size < min_part_size: # existing file size below minimal large file part size logger.debug( 'Fallback to full upload for %s -- remote file is smaller than %i bytes', self.local_path, min_part_size, ) return [self] if self.get_content_length() < file_version.size: logger.debug( 'Fallback to full upload for %s -- local file is smaller than remote', self.local_path, ) return [self] content_sha1 = file_version.get_content_sha1() if not content_sha1: logger.debug( 'Fallback to full upload for %s -- remote file content SHA1 unknown', self.local_path, ) return [self] # We're calculating hexdigest of the first N bytes of the file. However, if the sha1 differs, # we'll be needing the whole hash of the file anyway. So we can use this partial information. with self.open() as fp: digester = IncrementalHexDigester(fp) hex_digest = digester.update_from_stream(file_version.size) if hex_digest != content_sha1: logger.debug( 'Fallback to full upload for %s -- content in common range differs', self.local_path, ) # Calculate SHA1 of the remainder of the file and set it. self.content_sha1 = digester.update_from_stream() return [self] logger.debug('Incremental upload of %s is possible.', self.local_path) if file_version.server_side_encryption and file_version.server_side_encryption.is_unknown(): source_encryption = None else: source_encryption = file_version.server_side_encryption sources = [ CopySource( file_version.id_, offset=0, length=file_version.size, encryption=source_encryption, source_file_info=file_version.file_info, source_content_type=file_version.content_type, ), UploadSourceLocalFileRange(self.local_path, offset=file_version.size), ] return sources class UploadSourceStream(AbstractUploadSource): def __init__( self, stream_opener: Callable[[], io.IOBase], stream_length: int | None = None, stream_sha1: Sha1HexDigest | None = None, ): """ Initialize upload source using arbitrary function. :param stream_opener: A function that opens a stream for uploading. :param stream_length: Length of the stream. If ``None``, data will be calculated from the stream the first time it's required. :param stream_sha1: SHA1 of the stream. If ``None``, data will be calculated from the stream the first time it's required. """ self.stream_opener = stream_opener self._content_length = stream_length self._content_sha1 = stream_sha1 def __repr__(self) -> str: return ( f'<{self.__class__.__name__} stream_opener={repr(self.stream_opener)} content_length={self._content_length} ' f'content_sha1={self._content_sha1} id={id(self)}>' ) def get_content_length(self) -> int: if self._content_length is None: self._set_content_length_and_sha1() return self._content_length def get_content_sha1(self) -> Sha1HexDigest | None: if self._content_sha1 is None: self._set_content_length_and_sha1() return self._content_sha1 def open(self): return self.stream_opener() def _set_content_length_and_sha1(self) -> None: sha1, content_length = hex_sha1_of_unlimited_stream(self.open()) self._content_length = content_length self._content_sha1 = sha1 def is_sha1_known(self) -> bool: return self._content_sha1 is not None class UploadSourceStreamRange(UploadSourceStream): def __init__( self, stream_opener: Callable[[], io.IOBase], offset: int = 0, stream_length: int | None = None, stream_sha1: Sha1HexDigest | None = None, ): """ Initialize upload source using arbitrary function. :param stream_opener: A function that opens a stream for uploading. :param offset: Offset from which stream should be uploaded. :param stream_length: Length of the stream. If ``None``, data will be calculated from the stream the first time it's required. :param stream_sha1: SHA1 of the stream. If ``None``, data will be calculated from the stream the first time it's required. """ super().__init__( stream_opener, stream_length=stream_length, stream_sha1=stream_sha1, ) self._offset = offset def __repr__(self) -> str: return ( f'<{self.__class__.__name__} stream_opener={repr(self.stream_opener)} offset={self._offset} ' f'content_length={self._content_length} content_sha1={self._content_sha1} id={id(self)}>' ) def open(self): return RangeOfInputStream(super().open(), self._offset, self._content_length) b2-sdk-python-2.8.0/b2sdk/_internal/transfer/transfer_manager.py000066400000000000000000000010601474454370000246230ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/transfer/transfer_manager.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations class TransferManager: """ Base class for manager classes (copy, upload, download) """ def __init__(self, services, **kwargs): self.services = services super().__init__(**kwargs) b2-sdk-python-2.8.0/b2sdk/_internal/types.py000066400000000000000000000021241474454370000206270ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/types.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### """ Types compatibility layer. We use this module to support pydantic-less installs, as well as native typing module us on newer python versions. """ import sys from annotated_types import Ge try: from typing_extensions import Annotated, NotRequired, TypedDict except ImportError: from typing import Annotated, NotRequired, TypedDict __all__ = [ # prevents linter from removing "unused imports" which we want to export 'NotRequired', 'PositiveInt', 'TypedDict', 'pydantic', ] try: import pydantic if getattr(pydantic, '__version__', '') < '2': raise ImportError if sys.version_info < (3, 10): # https://github.com/pydantic/pydantic/issues/7873 import eval_type_backport # noqa except ImportError: pydantic = None PositiveInt = Annotated[int, Ge(0)] b2-sdk-python-2.8.0/b2sdk/_internal/utils/000077500000000000000000000000001474454370000202525ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_internal/utils/__init__.py000066400000000000000000000340611474454370000223670ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/utils/__init__.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import base64 import hashlib import logging import os import pathlib import platform import re import time from dataclasses import dataclass, field from decimal import Decimal from itertools import chain from typing import Any, Iterator, NewType, TypeVar from urllib.parse import quote, unquote_plus from logfury.v1 import ( DefaultTraceAbstractMeta, DefaultTraceMeta, limit_trace_arguments, disable_trace, trace_call, ) logger = logging.getLogger(__name__) Sha1HexDigest = NewType('Sha1HexDigest', str) T = TypeVar('T') # TODO: When we drop Python 3.7 support, this should be replaced # with typing.Protocol that exposes read method. ReadOnlyStream = Any def b2_url_encode(s): """ URL-encode a unicode string to be sent to B2 in an HTTP header. :param s: a unicode string to encode :type s: str :return: URL-encoded string :rtype: str """ return quote(s.encode('utf-8')) def b2_url_decode(s): """ Decode a Unicode string returned from B2 in an HTTP header. :param s: a unicode string to decode :type s: str :return: a Python unicode string. :rtype: str """ return unquote_plus(s) def choose_part_ranges(content_length, minimum_part_size): """ Return a list of (offset, length) for the parts of a large file. :param content_length: content length value :type content_length: int :param minimum_part_size: a minimum file part size :type minimum_part_size: int :rtype: list """ # If the file is at least twice the minimum part size, we are guaranteed # to be able to break it into multiple parts that are all at least # the minimum part size. assert minimum_part_size * 2 <= content_length # How many parts can we make? part_count = min(content_length // minimum_part_size, 10000) assert 2 <= part_count # All of the parts, except the last, are the same size. The # last one may be bigger. part_size = content_length // part_count last_part_size = content_length - (part_size * (part_count - 1)) assert minimum_part_size <= last_part_size # Make all of the parts except the last parts = [(i * part_size, part_size) for i in range(part_count - 1)] # Add the last part start_of_last = (part_count - 1) * part_size last_part = (start_of_last, content_length - start_of_last) parts.append(last_part) return parts def update_digest_from_stream(digest: T, input_stream: ReadOnlyStream, content_length: int) -> T: """ Update and return `digest` with data read from `input_stream` :param digest: a digest object, which exposes an `update(bytes)` method :param input_stream: stream object, which exposes a `read(int|None)` method :param content_length: expected length of the stream :type content_length: int """ remaining = content_length block_size = 1024 * 1024 while remaining != 0: to_read = min(remaining, block_size) data = input_stream.read(to_read) if len(data) != to_read: raise ValueError( 'content_length(%s) is more than the size of the file' % content_length ) digest.update(data) remaining -= to_read return digest def hex_sha1_of_stream(input_stream: ReadOnlyStream, content_length: int) -> Sha1HexDigest: """ Return the 40-character hex SHA1 checksum of the first content_length bytes in the input stream. :param input_stream: stream object, which exposes read(int|None) method :param content_length: expected length of the stream :type content_length: int :rtype: str """ return Sha1HexDigest( update_digest_from_stream( hashlib.sha1(), input_stream, content_length, ).hexdigest() ) @dataclass class IncrementalHexDigester: """ Calculates digest of a stream or parts of it. """ stream: ReadOnlyStream digest: 'hashlib._Hash' = field( # noqa (_Hash is a dynamic object) default_factory=hashlib.sha1 ) read_bytes: int = 0 block_size: int = 1024 * 1024 @property def hex_digest(self) -> Sha1HexDigest: return Sha1HexDigest(self.digest.hexdigest()) def update_from_stream( self, limit: int | None = None, ) -> Sha1HexDigest: """ :param limit: How many new bytes try to read from the stream. Default None – read until nothing left. """ offset = 0 while True: if limit is not None: to_read = min(limit - offset, self.block_size) else: to_read = self.block_size data = self.stream.read(to_read) data_len = len(data) if data_len > 0: self.digest.update(data) self.read_bytes += data_len offset += data_len if data_len < to_read or to_read == 0: break return self.hex_digest def hex_sha1_of_unlimited_stream( input_stream: ReadOnlyStream, limit: int | None = None, ) -> tuple[Sha1HexDigest, int]: digester = IncrementalHexDigester(input_stream) digester.update_from_stream(limit) return digester.hex_digest, digester.read_bytes def hex_sha1_of_file(path_) -> Sha1HexDigest: with open(path_, 'rb') as file: return hex_sha1_of_unlimited_stream(file)[0] def hex_sha1_of_bytes(data: bytes) -> Sha1HexDigest: """ Return the 40-character hex SHA1 checksum of the data. """ return Sha1HexDigest(hashlib.sha1(data).hexdigest()) def hex_md5_of_bytes(data: bytes) -> str: """ Return the 32-character hex MD5 checksum of the data. """ return hashlib.md5(data).hexdigest() def md5_of_bytes(data: bytes) -> bytes: """ Return the 16-byte MD5 checksum of the data. """ return hashlib.md5(data).digest() def b64_of_bytes(data: bytes) -> str: """ Return the base64 encoded represtantion of the data. """ return base64.b64encode(data).decode() def validate_b2_file_name(name): """ Raise a ValueError if the name is not a valid B2 file name. :param name: a string to check :type name: str """ if not isinstance(name, str): raise ValueError('file name must be a string, not bytes') try: name_utf8 = name.encode('utf-8') except UnicodeEncodeError: raise ValueError('file name must be valid Unicode, check locale') if len(name_utf8) < 1: raise ValueError('file name too short (0 utf-8 bytes)') if 1000 < len(name_utf8): raise ValueError('file name too long (more than 1000 utf-8 bytes)') if name[0] == '/': raise ValueError("file names must not start with '/'") if name[-1] == '/': raise ValueError("file names must not end with '/'") if '\\' in name: raise ValueError("file names must not contain '\\'") if '//' in name: raise ValueError("file names must not contain '//'") if chr(127) in name: raise ValueError('file names must not contain DEL') if any(250 < len(segment) for segment in name_utf8.split(b'/')): raise ValueError("file names segments (between '/') can be at most 250 utf-8 bytes") def get_file_mtime(local_path): """ Get modification time of a file in milliseconds. :param local_path: a file path :type local_path: str :rtype: int """ mod_time = os.path.getmtime(local_path) * 1000 return int(mod_time) def is_special_file(path: str | pathlib.Path) -> bool: """ Is the path a special file, such as /dev/null or stdout? :param path: a "file" path :return: True if the path is a special file """ path_str = str(path) return ( path == os.devnull or path_str.startswith('/dev/') or platform.system() == 'Windows' and path_str.upper() in ('CON', 'NUL') ) def set_file_mtime(local_path: str | pathlib.Path, mod_time_millis: int) -> None: """ Set modification time of a file in milliseconds. :param local_path: a file path :param mod_time_millis: time to be set """ mod_time = mod_time_millis / 1000.0 # We have to convert it this way to avoid differences when mtime # is read from the local file in the next iterations, and time is fetched # without rounding. # This is caused by floating point arithmetic as POSIX systems # represents mtime as floats and B2 as integers. # E.g. for 1093258377393, it would be converted to 1093258377.393 # which is actually represented by 1093258377.3929998874664306640625. # When we save mtime and read it again, we will end up with 1093258377392. # See #617 for details. mod_time = float(Decimal('%.3f5' % mod_time)) try: os.utime(local_path, (mod_time, mod_time)) except OSError: if not is_special_file(local_path): raise def fix_windows_path_limit(path): """ Prefix paths when running on Windows to overcome 260 character path length limit. See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath :param path: a path to prefix :type path: str :return: a prefixed path :rtype: str """ if platform.system() == 'Windows': if path.startswith('\\\\'): # UNC network path return '\\\\?\\UNC\\' + path[2:] elif os.path.isabs(path): # local absolute path return '\\\\?\\' + path else: # relative path, don't alter return path else: return path def _pick_scale_and_suffix(x): # suffixes for different scales suffixes = ' kMGTP' # We want to use the biggest suffix that makes sense. ref_digits = str(int(x)) index = (len(ref_digits) - 1) // 3 suffix = suffixes[index] if suffix == ' ': suffix = '' scale = 1000**index return (scale, suffix) def format_and_scale_number(x, unit): """ Pick a good scale for representing a number and format it. :param x: a number :type x: int :param unit: an arbitrary unit name :type unit: str :return: scaled and formatted number :rtype: str """ # simple case for small numbers if x < 1000: return '%d %s' % (x, unit) # pick a scale (scale, suffix) = _pick_scale_and_suffix(x) # decide how many digits after the decimal to display scaled = x / scale if scaled < 10.0: fmt = '%1.2f %s%s' elif scaled < 100.0: fmt = '%1.1f %s%s' else: fmt = '%1.0f %s%s' # format it return fmt % (scaled, suffix, unit) def format_and_scale_fraction(numerator, denominator, unit): """ Pick a good scale for representing a fraction, and format it. :param numerator: a numerator of a fraction :type numerator: int :param denominator: a denominator of a fraction :type denominator: int :param unit: an arbitrary unit name :type unit: str :return: scaled and formatted fraction :rtype: str """ # simple case for small numbers if denominator < 1000: return '%d / %d %s' % (numerator, denominator, unit) # pick a scale (scale, suffix) = _pick_scale_and_suffix(denominator) # decide how many digits after the decimal to display scaled_denominator = denominator / scale if scaled_denominator < 10.0: fmt = '%1.2f / %1.2f %s%s' elif scaled_denominator < 100.0: fmt = '%1.1f / %1.1f %s%s' else: fmt = '%1.0f / %1.0f %s%s' # format it scaled_numerator = numerator / scale return fmt % (scaled_numerator, scaled_denominator, suffix, unit) _CAMELCASE_TO_UNDERSCORE_RE = re.compile('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))') def camelcase_to_underscore(input_): """ Convert a camel-cased string to a string with underscores. :param input_: an input string :type input_: str :return: string with underscores :rtype: str """ return _CAMELCASE_TO_UNDERSCORE_RE.sub(r'_\1', input_).lower() class B2TraceMeta(DefaultTraceMeta): """ Trace all public method calls, except for ones with names that begin with `get_`. """ pass class B2TraceMetaAbstract(DefaultTraceAbstractMeta): """ Default class for tracers, to be set as a metaclass for abstract base classes. """ pass class ConcurrentUsedAuthTokenGuard: """ Context manager preventing two tokens being used simultaneously. Throws UploadTokenUsedConcurrently when unable to acquire a lock Sample usage: with ConcurrentUsedAuthTokenGuard(lock_for_token, token): # code that uses the token exclusively """ def __init__(self, lock, token): self.lock = lock self.token = token def __enter__(self): if not self.lock.acquire(False): from b2sdk._internal.exception import UploadTokenUsedConcurrently raise UploadTokenUsedConcurrently(self.token) def __exit__(self, exc_type, exc_val, exc_tb): try: self.lock.release() except RuntimeError: # guard against releasing a non-acquired lock pass def current_time_millis(): """ File times are in integer milliseconds, to avoid roundoff errors. """ return int(round(time.time() * 1000)) def iterator_peek(iterator: Iterator[T], count: int) -> tuple[list[T], Iterator[T]]: """ Get up to the `count` first elements yielded by `iterator`. The function will read `count` elements from `iterator` or less if the end is reached first. Returns a tuple consisting of a list of retrieved elements and an iterator equivalent to the input iterator. """ ret = [] for _ in range(count): try: ret.append(next(iterator)) except StopIteration: break return ret, chain(ret, iterator) assert disable_trace assert limit_trace_arguments assert trace_call b2-sdk-python-2.8.0/b2sdk/_internal/utils/docs.py000066400000000000000000000030321474454370000215520ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/utils/docs.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations class MissingDocURL(Exception): pass def ensure_b2sdk_doc_urls(cls: type): """ Decorator to indicate (and verify) that class has external documentation URLs. Used for to validate that all classes have external documentation URLs properly defined. """ urls = get_b2sdk_doc_urls(cls) if not urls: raise MissingDocURL(f'No documentation URLs found for {cls.__name__}') return cls def get_b2sdk_doc_urls(type_: type) -> dict[str, str]: """ Get the external documentation URLs for a b2sdk class. Non-b2sdk classes are not, and will not be supported. :param type_: the class to get the documentation URLs for :return: a dictionary mapping link names to URLs """ docstring = type_.__doc__ if not docstring: return {} return _extract_restructedtext_links(docstring) _rest_link_prefix = '.. _' def _extract_restructedtext_links(docstring: str) -> dict[str, str]: links = {} for line in docstring.splitlines(): line = line.strip() if line.startswith(_rest_link_prefix): name, url = line[len(_rest_link_prefix) :].split(': ', 1) if name and url: links[name] = url return links b2-sdk-python-2.8.0/b2sdk/_internal/utils/escape.py000066400000000000000000000031731474454370000220700ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/utils/escape.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import re import shlex # skip newline, tab UNPRINTABLE_PATTERN = re.compile(r'[\x00-\x08\x0e-\x1f\x7f-\x9f]') def unprintable_to_hex(s: str) -> str: """ Replace unprintable chars in string with a hex representation. :param s: an arbitrary string, possibly with unprintable characters. :return: the string, with unprintable characters changed to hex (e.g., "\x07") """ def hexify(match): return rf'\x{ord(match.group()):02x}' if s: return UNPRINTABLE_PATTERN.sub(hexify, s) return s def escape_control_chars(s: str) -> str: """ Replace unprintable chars in string with a hex representation AND shell quotes the string. :param s: an arbitrary string, possibly with unprintable characters. :return: the string, with unprintable characters changed to hex (e.g., "\x07") """ if s: return shlex.quote(unprintable_to_hex(s)) return s def substitute_control_chars(s: str) -> tuple[str, bool]: """ Replace unprintable chars in string with � unicode char :param s: an arbitrary string, possibly with unprintable characters. :return: tuple of the string with � replacements made and boolean indicated if chars were replaced """ new_value = UNPRINTABLE_PATTERN.sub('�', s) return new_value, new_value != s b2-sdk-python-2.8.0/b2sdk/_internal/utils/filesystem.py000066400000000000000000000016631474454370000230160ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/utils/filesystem.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pathlib import platform import stat _IS_WINDOWS = platform.system() == 'Windows' def points_to_fifo(path: pathlib.Path) -> bool: """Check if the path points to a fifo.""" path = path.resolve() try: return stat.S_ISFIFO(path.stat().st_mode) except OSError: return False _STDOUT_FILENAME = 'CON' if _IS_WINDOWS else '/dev/stdout' STDOUT_FILEPATH = pathlib.Path(_STDOUT_FILENAME) def points_to_stdout(path: pathlib.Path) -> bool: """Check if the path points to stdout.""" try: return path == STDOUT_FILEPATH or path.resolve() == STDOUT_FILEPATH except OSError: return False b2-sdk-python-2.8.0/b2sdk/_internal/utils/http_date.py000066400000000000000000000022421474454370000226000ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/utils/http_date.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import datetime as dt def parse_http_date(timestamp_str: str) -> dt.datetime: # See https://datatracker.ietf.org/doc/html/rfc7231#section-7.1.1.1 # for the list of supported formats. # We don't parse non-GTM dates because they are not valid HTTP-dates # as defined in RFC7231 7.1.1.1. Backblaze is more premissive than # the standard here. http_data_formats = [ '%a, %d %b %Y %H:%M:%S GMT', # IMF-fixdate '%A, %d-%b-%y %H:%M:%S GMT', # obsolete RFC 850 format '%a %b %d %H:%M:%S %Y', # ANSI C's asctime() format ] for format in http_data_formats: try: timestamp = dt.datetime.strptime(timestamp_str, format) return timestamp.replace(tzinfo=dt.timezone.utc) except ValueError: pass raise ValueError("Value %s is not a valid HTTP-date, won't be parsed.", timestamp_str) b2-sdk-python-2.8.0/b2sdk/_internal/utils/range_.py000066400000000000000000000047461474454370000220720ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/utils/range_.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import dataclasses import re _RANGE_HEADER_RE = re.compile( r'^(?:bytes[ =])?(?P\d+)-(?P\d+)(?:/(?:(?P\d+)|\*))?$' ) @dataclasses.dataclass(eq=True, order=True, frozen=True) class Range: """ HTTP ranges use an *inclusive* index at the end. """ __slots__ = ['start', 'end'] start: int end: int def __post_init__(self): assert 0 <= self.start <= self.end or ( self.start == 1 and self.end == 0 ), f'Invalid range: {self}' @classmethod def from_header(cls, raw_range_header: str) -> Range: """ Factory method which returns an object constructed from Range http header. raw_range_header example: 'bytes=0-11' """ return cls.from_header_with_size(raw_range_header)[0] @classmethod def from_header_with_size(cls, raw_range_header: str) -> tuple[Range, int | None]: """ Factory method which returns an object constructed from Range http header. raw_range_header example: 'bytes=0-11' """ match = _RANGE_HEADER_RE.match(raw_range_header) if not match: raise ValueError(f'Invalid range header: {raw_range_header}') start = int(match.group('start')) end = int(match.group('end')) complete_length = match.group('complete_length') complete_length = int(complete_length) if complete_length else None return cls(start, end), complete_length def size(self) -> int: return self.end - self.start + 1 def subrange(self, sub_start, sub_end) -> Range: """ Return a range that is part of this range. :param sub_start: index relative to the start of this range. :param sub_end: (Inclusive!) index relative to the start of this range. :return: a new Range """ assert 0 <= sub_start <= sub_end < self.size() return self.__class__(self.start + sub_start, self.start + sub_end) def as_tuple(self) -> tuple[int, int]: return self.start, self.end def __repr__(self) -> str: return f'{self.__class__.__name__}({self.start}, {self.end})' EMPTY_RANGE = Range(1, 0) b2-sdk-python-2.8.0/b2sdk/_internal/utils/thread_pool.py000066400000000000000000000067361474454370000231400ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/utils/thread_pool.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import os from concurrent.futures import Future, ThreadPoolExecutor from typing import Callable try: from typing_extensions import Protocol except ImportError: from typing import Protocol from b2sdk._internal.utils import B2TraceMetaAbstract class DynamicThreadPoolExecutorProtocol(Protocol): def submit(self, fn: Callable, *args, **kwargs) -> Future: ... def set_size(self, max_workers: int) -> None: """Set the size of the thread pool.""" def get_size(self) -> int: """Return the current size of the thread pool.""" class LazyThreadPool: """ Lazily initialized thread pool. """ _THREAD_POOL_FACTORY = ThreadPoolExecutor def __init__(self, max_workers: int | None = None, **kwargs): if max_workers is None: max_workers = min( 32, (os.cpu_count() or 1) + 4 ) # same default as in ThreadPoolExecutor self._max_workers = max_workers self._thread_pool: ThreadPoolExecutor | None = None super().__init__(**kwargs) def submit(self, fn: Callable, *args, **kwargs) -> Future: if self._thread_pool is None: self._thread_pool = self._THREAD_POOL_FACTORY(self._max_workers) return self._thread_pool.submit(fn, *args, **kwargs) def set_size(self, max_workers: int) -> None: """ Set the size of the thread pool. This operation will block until all tasks in the current thread pool are completed. :param max_workers: New size of the thread pool :return: None """ if self._max_workers == max_workers: return old_thread_pool = self._thread_pool self._thread_pool = self._THREAD_POOL_FACTORY(max_workers=max_workers) if old_thread_pool is not None: old_thread_pool.shutdown(wait=True) self._max_workers = max_workers def get_size(self) -> int: """Return the current size of the thread pool.""" return self._max_workers class ThreadPoolMixin(metaclass=B2TraceMetaAbstract): """ Mixin class with ThreadPoolExecutor. """ DEFAULT_THREAD_POOL_CLASS = LazyThreadPool def __init__( self, thread_pool: DynamicThreadPoolExecutorProtocol | None = None, max_workers: int | None = None, **kwargs, ): """ :param thread_pool: thread pool to be used :param max_workers: maximum number of worker threads (ignored if thread_pool is not None) """ self._thread_pool = ( thread_pool if thread_pool is not None else self.DEFAULT_THREAD_POOL_CLASS(max_workers=max_workers) ) self._max_workers = max_workers super().__init__(**kwargs) def set_thread_pool_size(self, max_workers: int) -> None: """ Set the size of the thread pool. This operation will block until all tasks in the current thread pool are completed. :param max_workers: New size of the thread pool :return: None """ return self._thread_pool.set_size(max_workers) def get_thread_pool_size(self) -> int: return self._thread_pool.get_size() b2-sdk-python-2.8.0/b2sdk/_internal/utils/typing.py000066400000000000000000000010611474454370000221340ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/utils/typing.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from typing import Dict, List, Union try: from typing_extensions import TypeAlias except ImportError: from typing import TypeAlias JSON: TypeAlias = Union[Dict[str, 'JSON'], List['JSON'], str, int, float, bool, None] b2-sdk-python-2.8.0/b2sdk/_internal/version_utils.py000066400000000000000000000154411474454370000223760ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_internal/version_utils.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import inspect import re import warnings from abc import ABCMeta, abstractmethod from functools import total_ordering, wraps from b2sdk.version import VERSION @total_ordering class _Version: """ Rudimentary semver version parser. It uses VERY naive parsing which is only supposed to produce a tuple, able to compare major.minor.patch versions. It does not support PEP 440 epoch, pre-releases, post-releases, local versions, etc. """ def __init__(self, version: str): self._raw = version self._parsed = self._parse_version(version) def __str__(self): return self._raw def __eq__(self, other): return self._parsed == other._parsed def __lt__(self, other): return self._parsed < other._parsed @classmethod def _parse_version(cls, version: str) -> tuple[int, ...]: if '!' in version: # strip PEP 440 epoch version = version.split('!', 1)[1] return tuple(map(int, re.findall(r'\d+', version))) class AbstractVersionDecorator(metaclass=ABCMeta): WHAT = NotImplemented # 'function', 'method', 'class' etc def __init__(self, changed_version, cutoff_version=None, reason='', current_version=None): """ Changed_version, cutoff_version and current_version are version strings. """ if current_version is None: # this is for tests only current_version = VERSION # TODO autodetect by going up the qualname tree and trying getattr(part, '__version__') self.current_version = _Version(current_version) #: current version self.reason = reason self.changed_version = self._parse_if_not_none( changed_version ) #: version in which the decorator was added self.cutoff_version = self._parse_if_not_none( cutoff_version ) #: version in which the decorator (and something?) shall be removed @classmethod def _parse_if_not_none(cls, version): if version is None: return None return _Version(version) @abstractmethod def __call__(self, func): """ The actual implementation of decorator. Needs self.source to be set before it's called. """ if self.cutoff_version and self.changed_version: assert ( self.changed_version < self.cutoff_version ), f'{self.__class__.__name__} decorator is set to start renaming {self.WHAT} {self.source!r} starting at version {self.changed_version} and finishing in {self.cutoff_version}. It needs to start at a lower version and finish at a higher version.' class AbstractDeprecator(AbstractVersionDecorator): ALTERNATIVE_DECORATOR = NotImplemented def __init__(self, target, *args, **kwargs): self.target = target super().__init__(*args, **kwargs) class rename_argument(AbstractDeprecator): """ Change the argument name to new one if old one is used, warns about deprecation in docs and through a warning. >>> @rename_argument('aaa', 'bbb', '0.1.0', '0.2.0') >>> def easy(bbb): >>> return bbb >>> easy(aaa=5) 'aaa' is a deprecated argument for 'easy' function/method - it was renamed to 'bbb' in version 0.1.0. Support for the old name is going to be dropped in 0.2.0. 5 >>> """ WHAT = 'argument' ALTERNATIVE_DECORATOR = 'discourage_argument' def __init__(self, source, *args, **kwargs): self.source = source super().__init__(*args, **kwargs) def __call__(self, func): super().__call__(func) signature = inspect.signature(func) has_target_arg = self.target in signature.parameters or any( p.kind == p.VAR_KEYWORD for p in signature.parameters.values() ) assert has_target_arg, f'{self.target!r} is not an argument of the decorated function so it cannot be remapped to from a deprecated parameter name' @wraps(func) def wrapper(*args, **kwargs): if self.source in kwargs: assert ( self.target not in kwargs ), f'both argument names were provided: {self.source!r} (deprecated) and {self.target!r} (new)' kwargs[self.target] = kwargs[self.source] del kwargs[self.source] info = f'{self.source!r} is a deprecated argument for {func.__name__!r} function/method - it was renamed to {self.target!r}' if self.changed_version: info += f' in version {self.changed_version}' if self.cutoff_version: info += f'. Support for the old name is going to be dropped in {self.cutoff_version}' warnings.warn( f'{info}.', DeprecationWarning, ) return func(*args, **kwargs) return wrapper class rename_function(AbstractDeprecator): """ Warn about deprecation in docs and through a DeprecationWarning when used. Use it to decorate a proxy function, like this: >>> def new(foobar): >>> return foobar ** 2 >>> @rename_function(new, '0.1.0', '0.2.0') >>> def old(foo, bar): >>> return new(foo + bar) >>> old() 'old' is deprecated since version 0.1.0 - it was moved to 'new', please switch to use that. The proxy for the old name is going to be removed in 0.2.0. 123 >>> """ WHAT = 'function' ALTERNATIVE_DECORATOR = 'discourage_function' def __init__(self, target, *args, **kwargs): if callable(target): target = target.__name__ super().__init__(target, *args, **kwargs) def __call__(self, func): self.source = func.__name__ super().__call__(func) @wraps(func) def wrapper(*args, **kwargs): warnings.warn( f'{func.__name__!r} is deprecated since version {self.changed_version} - it was moved to {self.target!r}, please switch to use that. The proxy for the old name is going to be removed in {self.cutoff_version}.', DeprecationWarning, ) return func(*args, **kwargs) return wrapper class rename_method(rename_function): WHAT = 'method' ALTERNATIVE_DECORATOR = 'discourage_method' class FeaturePreviewWarning(FutureWarning): """ Feature Preview Warning Marks a feature, that is in "Feature Preview" state. Such features are not yet fully stable and are subject to change or even outright removal. Do not rely on them in production code. """ b2-sdk-python-2.8.0/b2sdk/_pyinstaller/000077500000000000000000000000001474454370000176445ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_pyinstaller/__init__.py000066400000000000000000000010471474454370000217570ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_pyinstaller/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import os def get_hook_dirs(): """Get hooks directories for pyinstaller. More info about the hooks: https://pyinstaller.readthedocs.io/en/stable/hooks.html """ return [os.path.dirname(__file__)] b2-sdk-python-2.8.0/b2sdk/_pyinstaller/hook-b2sdk.py000066400000000000000000000006411474454370000221620ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_pyinstaller/hook-b2sdk.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from PyInstaller.utils.hooks import copy_metadata datas = copy_metadata('b2sdk') b2-sdk-python-2.8.0/b2sdk/_v3/000077500000000000000000000000001474454370000156265ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/_v3/__init__.py000066400000000000000000000304661474454370000177500ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_v3/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # Set default logging handler to avoid "No handler found" warnings. import logging as _logging _logging.getLogger('b2sdk').addHandler(_logging.NullHandler()) class UrllibWarningFilter: def filter(self, record): return record.msg != 'Connection pool is full, discarding connection: %s' _logging.getLogger('urllib3.connectionpool').addFilter(UrllibWarningFilter()) # this file maps the external interface into internal interface # it will come handy if we ever need to move something # core from b2sdk._internal.api import B2Api from b2sdk._internal.api import Services from b2sdk._internal.bucket import Bucket from b2sdk._internal.bucket import BucketFactory from b2sdk._internal.raw_api import ALL_CAPABILITIES, REALM_URLS, EVENT_TYPE # encryption from b2sdk._internal.encryption.setting import EncryptionSetting from b2sdk._internal.encryption.setting import EncryptionSettingFactory from b2sdk._internal.encryption.setting import EncryptionKey from b2sdk._internal.encryption.setting import SSE_NONE, SSE_B2_AES, UNKNOWN_KEY_ID from b2sdk._internal.encryption.types import EncryptionAlgorithm from b2sdk._internal.encryption.types import EncryptionMode from b2sdk._internal.http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME # account info from b2sdk._internal.account_info.abstract import AbstractAccountInfo from b2sdk._internal.account_info.in_memory import InMemoryAccountInfo from b2sdk._internal.account_info.sqlite_account_info import SqliteAccountInfo from b2sdk._internal.account_info.sqlite_account_info import B2_ACCOUNT_INFO_ENV_VAR from b2sdk._internal.account_info.sqlite_account_info import B2_ACCOUNT_INFO_DEFAULT_FILE from b2sdk._internal.account_info.sqlite_account_info import B2_ACCOUNT_INFO_PROFILE_FILE from b2sdk._internal.account_info.sqlite_account_info import XDG_CONFIG_HOME_ENV_VAR from b2sdk._internal.account_info.stub import StubAccountInfo from b2sdk._internal.account_info.upload_url_pool import UploadUrlPool from b2sdk._internal.account_info.upload_url_pool import UrlPoolAccountInfo # version & version utils from b2sdk.version import VERSION, USER_AGENT from b2sdk._internal.version_utils import rename_argument, rename_function, FeaturePreviewWarning # utils from b2sdk._internal.utils import ( b2_url_encode, b2_url_decode, choose_part_ranges, current_time_millis, fix_windows_path_limit, format_and_scale_fraction, format_and_scale_number, hex_sha1_of_stream, hex_sha1_of_bytes, hex_sha1_of_file, IncrementalHexDigester, ) from b2sdk._internal.utils.filesystem import ( points_to_fifo, points_to_stdout, STDOUT_FILEPATH, ) from b2sdk._internal.utils import trace_call from b2sdk._internal.utils.docs import get_b2sdk_doc_urls # data classes from b2sdk._internal.application_key import ApplicationKey from b2sdk._internal.application_key import BaseApplicationKey from b2sdk._internal.application_key import FullApplicationKey from b2sdk._internal.file_version import DownloadVersion from b2sdk._internal.file_version import DownloadVersionFactory from b2sdk._internal.file_version import FileIdAndName from b2sdk._internal.file_version import FileVersion from b2sdk._internal.file_version import FileVersionFactory from b2sdk._internal.large_file.part import Part from b2sdk._internal.large_file.unfinished_large_file import UnfinishedLargeFile from b2sdk._internal.large_file.services import LargeFileServices from b2sdk._internal.utils.range_ import Range, EMPTY_RANGE # file lock from b2sdk._internal.file_lock import BucketRetentionSetting from b2sdk._internal.file_lock import FileLockConfiguration from b2sdk._internal.file_lock import FileRetentionSetting from b2sdk._internal.file_lock import LegalHold from b2sdk._internal.file_lock import NO_RETENTION_BUCKET_SETTING from b2sdk._internal.file_lock import NO_RETENTION_FILE_SETTING from b2sdk._internal.file_lock import RetentionMode from b2sdk._internal.file_lock import RetentionPeriod from b2sdk._internal.file_lock import UNKNOWN_BUCKET_RETENTION from b2sdk._internal.file_lock import UNKNOWN_FILE_LOCK_CONFIGURATION from b2sdk._internal.file_lock import UNKNOWN_FILE_RETENTION_SETTING # progress reporting from b2sdk._internal.progress import AbstractProgressListener from b2sdk._internal.progress import DoNothingProgressListener from b2sdk._internal.progress import ProgressListenerForTest from b2sdk._internal.progress import SimpleProgressListener from b2sdk._internal.progress import TqdmProgressListener from b2sdk._internal.progress import make_progress_listener # raw_simulator from b2sdk._internal.raw_simulator import BucketSimulator from b2sdk._internal.raw_simulator import FakeResponse from b2sdk._internal.raw_simulator import FileSimulator from b2sdk._internal.raw_simulator import KeySimulator from b2sdk._internal.raw_simulator import PartSimulator from b2sdk._internal.raw_simulator import RawSimulator # raw_api from b2sdk._internal.raw_api import AbstractRawApi from b2sdk._internal.raw_api import B2RawHTTPApi from b2sdk._internal.raw_api import MetadataDirectiveMode from b2sdk._internal.raw_api import LifecycleRule from b2sdk._internal.raw_api import ( NotificationRule, NotificationRuleResponse, notification_rule_response_to_request, ) # stream from b2sdk._internal.stream.chained import StreamOpener from b2sdk._internal.stream.progress import AbstractStreamWithProgress from b2sdk._internal.stream import RangeOfInputStream from b2sdk._internal.stream import ReadingStreamWithProgress from b2sdk._internal.stream import StreamWithHash from b2sdk._internal.stream import WritingStreamWithProgress # source / destination from b2sdk._internal.transfer.inbound.downloaded_file import DownloadedFile from b2sdk._internal.transfer.inbound.downloaded_file import MtimeUpdatedFile from b2sdk._internal.transfer.inbound.download_manager import DownloadManager from b2sdk._internal.transfer.outbound.outbound_source import OutboundTransferSource from b2sdk._internal.transfer.outbound.copy_source import CopySource from b2sdk._internal.transfer.outbound.upload_source import AbstractUploadSource from b2sdk._internal.transfer.outbound.upload_source import UploadSourceBytes from b2sdk._internal.transfer.outbound.upload_source import UploadSourceLocalFile from b2sdk._internal.transfer.outbound.upload_source import UploadSourceLocalFileRange from b2sdk._internal.transfer.outbound.upload_source import UploadSourceStream from b2sdk._internal.transfer.outbound.upload_source import UploadSourceStreamRange from b2sdk._internal.transfer.outbound.upload_manager import UploadManager from b2sdk._internal.transfer.emerge.planner.upload_subpart import CachedBytesStreamOpener from b2sdk._internal.transfer.emerge.write_intent import WriteIntent # transfer from b2sdk._internal.transfer.inbound.downloader.abstract import AbstractDownloader from b2sdk._internal.transfer.outbound.large_file_upload_state import LargeFileUploadState from b2sdk._internal.transfer.inbound.downloader.parallel import ParallelDownloader from b2sdk._internal.transfer.inbound.downloader.parallel import PartToDownload from b2sdk._internal.transfer.inbound.downloader.parallel import WriterThread from b2sdk._internal.transfer.outbound.progress_reporter import PartProgressReporter from b2sdk._internal.transfer.inbound.downloader.simple import SimpleDownloader # sync from b2sdk._internal.sync.action import AbstractAction from b2sdk._internal.sync.action import B2CopyAction from b2sdk._internal.sync.action import B2DeleteAction from b2sdk._internal.sync.action import B2DownloadAction from b2sdk._internal.sync.action import B2HideAction from b2sdk._internal.sync.action import B2UploadAction from b2sdk._internal.sync.action import LocalDeleteAction from b2sdk._internal.sync.exception import IncompleteSync from b2sdk._internal.sync.policy import AbstractFileSyncPolicy from b2sdk._internal.sync.policy import CompareVersionMode from b2sdk._internal.sync.policy import NewerFileSyncMode from b2sdk._internal.sync.policy import DownAndDeletePolicy from b2sdk._internal.sync.policy import DownAndKeepDaysPolicy from b2sdk._internal.sync.policy import DownPolicy from b2sdk._internal.sync.policy import CopyPolicy from b2sdk._internal.sync.policy import CopyAndDeletePolicy from b2sdk._internal.sync.policy import CopyAndKeepDaysPolicy from b2sdk._internal.sync.policy import UpAndDeletePolicy from b2sdk._internal.sync.policy import UpAndKeepDaysPolicy from b2sdk._internal.sync.policy import UpPolicy from b2sdk._internal.sync.policy import make_b2_keep_days_actions from b2sdk._internal.sync.policy_manager import SyncPolicyManager from b2sdk._internal.sync.policy_manager import POLICY_MANAGER from b2sdk._internal.sync.report import SyncFileReporter from b2sdk._internal.sync.report import SyncReport from b2sdk._internal.sync.sync import KeepOrDeleteMode from b2sdk._internal.sync.sync import Synchronizer from b2sdk._internal.sync.sync import UploadMode from b2sdk._internal.sync.encryption_provider import AbstractSyncEncryptionSettingsProvider from b2sdk._internal.sync.encryption_provider import BasicSyncEncryptionSettingsProvider from b2sdk._internal.sync.encryption_provider import ServerDefaultSyncEncryptionSettingsProvider from b2sdk._internal.sync.encryption_provider import ( SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ) # scan from b2sdk._internal.scan.exception import EnvironmentEncodingError from b2sdk._internal.scan.exception import InvalidArgument from b2sdk._internal.scan.folder import AbstractFolder from b2sdk._internal.scan.folder import B2Folder from b2sdk._internal.scan.folder import LocalFolder from b2sdk._internal.scan.folder_parser import parse_folder from b2sdk._internal.scan.path import AbstractPath, B2Path, LocalPath from b2sdk._internal.scan.policies import convert_dir_regex_to_dir_prefix_regex from b2sdk._internal.scan.policies import DEFAULT_SCAN_MANAGER from b2sdk._internal.scan.policies import IntegerRange from b2sdk._internal.scan.policies import RegexSet from b2sdk._internal.scan.policies import ScanPoliciesManager from b2sdk._internal.scan.report import ProgressReport from b2sdk._internal.scan.scan import zip_folders from b2sdk._internal.scan.scan import AbstractScanResult from b2sdk._internal.scan.scan import AbstractScanReport from b2sdk._internal.scan.scan import CountAndSampleScanReport # replication from b2sdk._internal.replication.setting import ReplicationConfigurationFactory from b2sdk._internal.replication.setting import ReplicationConfiguration from b2sdk._internal.replication.setting import ReplicationRule from b2sdk._internal.replication.types import ReplicationStatus from b2sdk._internal.replication.setup import ReplicationSetupHelper from b2sdk._internal.replication.monitoring import ReplicationScanResult from b2sdk._internal.replication.monitoring import ReplicationReport from b2sdk._internal.replication.monitoring import ReplicationMonitor # other from b2sdk._internal.included_sources import get_included_sources from b2sdk._internal.b2http import B2Http from b2sdk._internal.api_config import B2HttpApiConfig from b2sdk._internal.api_config import DEFAULT_HTTP_API_CONFIG from b2sdk._internal.b2http import ClockSkewHook from b2sdk._internal.b2http import HttpCallback from b2sdk._internal.b2http import ResponseContextManager from b2sdk._internal.bounded_queue_executor import BoundedQueueExecutor from b2sdk._internal.cache import AbstractCache from b2sdk._internal.cache import AuthInfoCache from b2sdk._internal.cache import DummyCache from b2sdk._internal.cache import InMemoryCache from b2sdk._internal.http_constants import ( BUCKET_NAME_CHARS, BUCKET_NAME_CHARS_UNIQ, BUCKET_NAME_LENGTH_RANGE, DEFAULT_MAX_PART_SIZE, DEFAULT_MIN_PART_SIZE, DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE, LARGE_FILE_SHA1, LIST_FILE_NAMES_MAX_LIMIT, SRC_LAST_MODIFIED_MILLIS, ) from b2sdk._internal.session import B2Session from b2sdk._internal.utils.thread_pool import ThreadPoolMixin from b2sdk._internal.utils.escape import ( unprintable_to_hex, escape_control_chars, substitute_control_chars, ) # filter from b2sdk._internal.filter import FilterType, Filter # typing from b2sdk._internal.utils.typing import JSON b2-sdk-python-2.8.0/b2sdk/_v3/exception.py000066400000000000000000000115141474454370000202000ustar00rootroot00000000000000###################################################################### # # File: b2sdk/_v3/exception.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal.account_info.exception import AccountInfoError from b2sdk._internal.account_info.exception import CorruptAccountInfo from b2sdk._internal.account_info.exception import MissingAccountData from b2sdk._internal.exception import ( AccessDenied, AlreadyFailed, B2ConnectionError, B2Error, B2HttpCallbackException, B2HttpCallbackPostRequestException, B2HttpCallbackPreRequestException, B2RequestTimeout, B2RequestTimeoutDuringUpload, B2SimpleError, BadDateFormat, BadFileInfo, BadJson, BadRequest, BadUploadUrl, BrokenPipe, BucketIdNotFound, BucketNotAllowed, CapabilityNotAllowed, CapExceeded, ChecksumMismatch, ClockSkew, Conflict, ConnectionReset, CopyArgumentsMismatch, DestFileNewer, DestinationDirectoryDoesntAllowOperation, DestinationDirectoryDoesntExist, DestinationIsADirectory, DestinationParentIsNotADirectory, DisablingFileLockNotSupported, DuplicateBucketName, EmailNotVerified, FileAlreadyHidden, FileNameNotAllowed, FileNotPresent, FileSha1Mismatch, InvalidAuthToken, InvalidJsonResponse, InvalidMetadataDirective, InvalidRange, InvalidUploadSource, MaxFileSizeExceeded, MaxRetriesExceeded, MissingPart, NonExistentBucket, NoPaymentHistory, NotAllowedByAppKeyError, PartSha1Mismatch, PotentialS3EndpointPassedAsRealm, RestrictedBucket, RestrictedBucketMissing, RetentionWriteError, ServiceError, SourceReplicationConflict, SSECKeyError, SSECKeyIdMismatchInCopy, StorageCapExceeded, TooManyRequests, TransactionCapExceeded, TransientErrorMixin, TruncatedOutput, Unauthorized, UnexpectedCloudBehaviour, UnknownError, UnknownHost, UnrecognizedBucketType, UnsatisfiableRange, UnusableFileName, WrongEncryptionModeForBucketDefault, interpret_b2_error, ) from b2sdk._internal.scan.exception import EmptyDirectory from b2sdk._internal.scan.exception import EnvironmentEncodingError from b2sdk._internal.scan.exception import InvalidArgument from b2sdk._internal.scan.exception import NotADirectory from b2sdk._internal.scan.exception import UnableToCreateDirectory from b2sdk._internal.scan.exception import UnsupportedFilename from b2sdk._internal.scan.exception import check_invalid_argument from b2sdk._internal.sync.exception import IncompleteSync __all__ = ( 'AccessDenied', 'AccountInfoError', 'AlreadyFailed', 'B2ConnectionError', 'B2Error', 'B2HttpCallbackException', 'B2HttpCallbackPostRequestException', 'B2HttpCallbackPreRequestException', 'B2RequestTimeout', 'B2RequestTimeoutDuringUpload', 'B2SimpleError', 'BadDateFormat', 'BadFileInfo', 'BadJson', 'BadRequest', 'BadUploadUrl', 'BrokenPipe', 'BucketIdNotFound', 'BucketNotAllowed', 'CapabilityNotAllowed', 'CapExceeded', 'ChecksumMismatch', 'ClockSkew', 'Conflict', 'ConnectionReset', 'CopyArgumentsMismatch', 'CorruptAccountInfo', 'DestFileNewer', 'DestinationDirectoryDoesntAllowOperation', 'DestinationDirectoryDoesntExist', 'DestinationIsADirectory', 'DestinationParentIsNotADirectory', 'DisablingFileLockNotSupported', 'DuplicateBucketName', 'EmailNotVerified', 'EmptyDirectory', 'EnvironmentEncodingError', 'FileAlreadyHidden', 'FileNameNotAllowed', 'FileNotPresent', 'FileSha1Mismatch', 'IncompleteSync', 'InvalidArgument', 'InvalidAuthToken', 'InvalidJsonResponse', 'InvalidMetadataDirective', 'InvalidRange', 'InvalidUploadSource', 'MaxFileSizeExceeded', 'MaxRetriesExceeded', 'MissingAccountData', 'MissingPart', 'NonExistentBucket', 'NoPaymentHistory', 'NotADirectory', 'NotAllowedByAppKeyError', 'PartSha1Mismatch', 'PotentialS3EndpointPassedAsRealm', 'RestrictedBucket', 'RestrictedBucketMissing', 'RetentionWriteError', 'ServiceError', 'SourceReplicationConflict', 'StorageCapExceeded', 'TooManyRequests', 'TransactionCapExceeded', 'TransientErrorMixin', 'TruncatedOutput', 'UnableToCreateDirectory', 'Unauthorized', 'UnexpectedCloudBehaviour', 'UnknownError', 'UnknownHost', 'UnrecognizedBucketType', 'UnsatisfiableRange', 'UnsupportedFilename', 'UnusableFileName', 'interpret_b2_error', 'check_invalid_argument', 'SSECKeyIdMismatchInCopy', 'SSECKeyError', 'WrongEncryptionModeForBucketDefault', ) b2-sdk-python-2.8.0/b2sdk/v0/000077500000000000000000000000001474454370000154645ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/v0/__init__.py000066400000000000000000000013261474454370000175770ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk.v1 import * # noqa from b2sdk.v0.account_info import ( AbstractAccountInfo, InMemoryAccountInfo, UrlPoolAccountInfo, SqliteAccountInfo, ) from b2sdk.v0.api import B2Api from b2sdk.v0.bucket import Bucket from b2sdk.v0.bucket import BucketFactory from b2sdk.v0.sync import Synchronizer from b2sdk.v0.sync import make_folder_sync_actions from b2sdk.v0.sync import sync_folders b2-sdk-python-2.8.0/b2sdk/v0/account_info.py000066400000000000000000000037611474454370000205140ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/account_info.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal import version_utils from b2sdk import v1 class OldAccountInfoMethods: """this class contains proxy methods for deprecated signatures renamed for consistency in mid-2019""" def get_account_id_or_app_key_id(self): """ Return the application key ID used to authenticate. :rtype: str .. deprecated:: 0.1.6 Use :func:`get_application_key_id` instead. """ return self.get_application_key_id() @version_utils.rename_argument( 'account_id_or_app_key_id', 'application_key_id', '0.1.5', None, ) def set_auth_data( self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, allowed=None, application_key_id=None, s3_api_url=None, ): # we need to enumerate all the parameters and cannot just "*args, **kwargs" because # the deprecation decorator doesn't feel safe with the kwargs approach return super().set_auth_data( account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, allowed, application_key_id, s3_api_url=s3_api_url, ) class AbstractAccountInfo(OldAccountInfoMethods, v1.AbstractAccountInfo): pass class InMemoryAccountInfo(OldAccountInfoMethods, v1.InMemoryAccountInfo): pass class UrlPoolAccountInfo(OldAccountInfoMethods, v1.UrlPoolAccountInfo): pass class SqliteAccountInfo(OldAccountInfoMethods, v1.SqliteAccountInfo): pass b2-sdk-python-2.8.0/b2sdk/v0/api.py000066400000000000000000000017471474454370000166200ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/api.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from .bucket import Bucket, BucketFactory from b2sdk import v1 class B2Api(v1.B2Api): BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) BUCKET_CLASS = staticmethod(Bucket) def delete_bucket(self, bucket): """ Delete the chosen bucket. For legacy reasons it returns whatever server sends in response, but API user should not rely on the response: if it doesn't raise an exception, it means that the operation was a success. :param b2sdk.v1.Bucket bucket: a :term:`bucket` to delete """ account_id = self.account_info.get_account_id() return self.session.delete_bucket(account_id, bucket.id_) b2-sdk-python-2.8.0/b2sdk/v0/bucket.py000066400000000000000000000020661474454370000173170ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/bucket.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import v1 class Bucket(v1.Bucket): def list_file_names(self, start_filename=None, max_entries=None, prefix=None): """ Legacy interface which just returns whatever remote API returns. """ return self.api.session.list_file_names(self.id_, start_filename, max_entries, prefix) def list_file_versions( self, start_filename=None, start_file_id=None, max_entries=None, prefix=None ): """ Legacy interface which just returns whatever remote API returns. """ return self.api.session.list_file_versions( self.id_, start_filename, start_file_id, max_entries, prefix ) class BucketFactory(v1.BucketFactory): BUCKET_CLASS = staticmethod(Bucket) b2-sdk-python-2.8.0/b2sdk/v0/exception.py000066400000000000000000000015441474454370000200400ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations B2Error = None # calm down, pyflakes from b2sdk.v1.exception import * # noqa v1DestFileNewer = DestFileNewer # override to retain old style __str__ class DestFileNewer(v1DestFileNewer): def __str__(self): return f'source file is older than destination: {self.source_prefix}{self.source_file.name} with a time of {self.source_file.latest_version().mod_time} cannot be synced to {self.dest_prefix}{self.dest_file.name} with a time of {self.dest_file.latest_version().mod_time}, unless --skipNewer or --replaceNewer is provided' b2-sdk-python-2.8.0/b2sdk/v0/sync.py000066400000000000000000000161241474454370000170160ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v0/sync.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from b2sdk.v1.exception import CommandError from b2sdk.v1.exception import DestFileNewer as DestFileNewerV1 from b2sdk.v1 import trace_call from .exception import DestFileNewer from b2sdk.v1.exception import InvalidArgument, IncompleteSync from b2sdk.v1 import NewerFileSyncMode, CompareVersionMode from b2sdk.v1 import KeepOrDeleteMode from b2sdk.v1 import DEFAULT_SCAN_MANAGER from b2sdk.v1 import SyncReport from b2sdk.v1 import Synchronizer as SynchronizerV1 from b2sdk.v1 import ( AbstractSyncEncryptionSettingsProvider, SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ) logger = logging.getLogger(__name__) class Synchronizer(SynchronizerV1): """ This is wrapper for newer version and will return the v0 style exceptions """ def __init__(self, *args, **kwargs): try: super().__init__(*args, **kwargs) except InvalidArgument as e: raise CommandError(f'--{e.parameter_name} {e.message}') def _make_file_sync_actions(self, *args, **kwargs): try: yield from super()._make_file_sync_actions(*args, **kwargs) except DestFileNewerV1 as e: raise DestFileNewer(e.dest_file, e.source_file, e.dest_prefix, e.source_prefix) def sync_folders(self, *args, **kwargs): try: super().sync_folders(*args, **kwargs) except InvalidArgument as e: raise CommandError(f'--{e.parameter_name} {e.message}') except IncompleteSync as e: raise CommandError(str(e)) def get_synchronizer_from_args( args, max_workers, policies_manager=DEFAULT_SCAN_MANAGER, dry_run=False, allow_empty_source=False, ): if args.replaceNewer and args.skipNewer: raise CommandError('--skipNewer and --replaceNewer are incompatible') elif args.replaceNewer: newer_file_mode = NewerFileSyncMode.REPLACE elif args.skipNewer: newer_file_mode = NewerFileSyncMode.SKIP else: newer_file_mode = NewerFileSyncMode.RAISE_ERROR if args.delete and (args.keepDays is not None): raise CommandError('--delete and --keepDays are incompatible') if args.compareVersions == 'none': compare_version_mode = CompareVersionMode.NONE elif args.compareVersions == 'modTime': compare_version_mode = CompareVersionMode.MODTIME elif args.compareVersions == 'size': compare_version_mode = CompareVersionMode.SIZE elif args.compareVersions is None: compare_version_mode = CompareVersionMode.MODTIME else: raise CommandError('Invalid option for --compareVersions') compare_threshold = args.compareThreshold keep_days = None if args.delete: keep_days_or_delete = KeepOrDeleteMode.DELETE elif args.keepDays: keep_days_or_delete = KeepOrDeleteMode.KEEP_BEFORE_DELETE keep_days = args.keepDays else: keep_days_or_delete = KeepOrDeleteMode.NO_DELETE return Synchronizer( max_workers, policies_manager=policies_manager, dry_run=dry_run, allow_empty_source=allow_empty_source, newer_file_mode=newer_file_mode, keep_days_or_delete=keep_days_or_delete, compare_version_mode=compare_version_mode, compare_threshold=compare_threshold, keep_days=keep_days, ) def make_folder_sync_actions( source_folder, dest_folder, args, now_millis, reporter, policies_manager=DEFAULT_SCAN_MANAGER, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ This is deprecated. Use the new Synchronizer class. Yields a sequence of actions that will sync the destination folder to the source folder. :param source_folder: source folder object :type source_folder: b2sdk._internal.scan.folder.AbstractFolder :param dest_folder: destination folder object :type dest_folder: b2sdk._internal.scan.folder.AbstractFolder :param args: an object which holds command line arguments :param now_millis: current time in milliseconds :type now_millis: int :param reporter: reporter object :param policies_manager: policies manager object :param encryption_settings_provider: encryption settings provider :type encryption_settings_provider: AbstractSyncEncryptionSettingsProvider """ synchronizer = get_synchronizer_from_args( args, 1, policies_manager=policies_manager, dry_run=False, allow_empty_source=False, ) try: return synchronizer.make_folder_sync_actions( source_folder, dest_folder, now_millis, reporter, policies_manager=policies_manager, encryption_settings_provider=encryption_settings_provider, ) except InvalidArgument as e: raise CommandError(f'--{e.parameter_name} {e.message}') @trace_call(logger) def sync_folders( source_folder, dest_folder, args, now_millis, stdout, no_progress, max_workers, policies_manager=DEFAULT_SCAN_MANAGER, dry_run=False, allow_empty_source=False, encryption_settings_provider: AbstractSyncEncryptionSettingsProvider = SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ This is deprecated. Use the new Synchronizer class. source is also in the destination. Deletes any file versions in the destination older than history_days. :param source_folder: source folder object :type source_folder: b2sdk._internal.scan.folder.AbstractFolder :param dest_folder: destination folder object :type dest_folder: b2sdk._internal.scan.folder.AbstractFolder :param args: an object which holds command line arguments :param now_millis: current time in milliseconds :type now_millis: int :param stdout: standard output file object :param no_progress: if True, do not show progress :type no_progress: bool :param max_workers: max number of workers :type max_workers: int :param policies_manager: policies manager object :param dry_run: :type dry_run: bool :param allow_empty_source: if True, do not check whether source folder is empty :type allow_empty_source: bool :param encryption_settings_provider: encryption settings provider :type encryption_settings_provider: AbstractSyncEncryptionSettingsProvider """ synchronizer = get_synchronizer_from_args( args, max_workers, policies_manager=policies_manager, dry_run=dry_run, allow_empty_source=allow_empty_source, ) with SyncReport(stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder, dest_folder, now_millis, reporter, encryption_settings_provider=encryption_settings_provider, ) b2-sdk-python-2.8.0/b2sdk/v1/000077500000000000000000000000001474454370000154655ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/v1/__init__.py000066400000000000000000000026161474454370000176030ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk.v2 import * # noqa from b2sdk.v1.account_info import ( AbstractAccountInfo, InMemoryAccountInfo, UrlPoolAccountInfo, SqliteAccountInfo, StubAccountInfo, ) from b2sdk.v1.api import B2Api from b2sdk.v1.b2http import B2Http from b2sdk.v1.bucket import Bucket, BucketFactory from b2sdk.v1.cache import AbstractCache from b2sdk.v1.download_dest import ( AbstractDownloadDestination, DownloadDestLocalFile, PreSeekedDownloadDest, DownloadDestBytes, DownloadDestProgressWrapper, ) from b2sdk.v1.exception import CommandError, DestFileNewer from b2sdk.v1.file_metadata import FileMetadata from b2sdk.v1.file_version import FileVersionInfo from b2sdk.v1.session import B2Session from b2sdk.v1.sync import ( ScanPoliciesManager, DEFAULT_SCAN_MANAGER, zip_folders, Synchronizer, AbstractFolder, LocalFolder, B2Folder, parse_sync_folder, SyncReport, File, B2File, FileVersion, AbstractSyncEncryptionSettingsProvider, ) from b2sdk.v1.replication.monitoring import ReplicationMonitor B2RawApi = B2RawHTTPApi b2-sdk-python-2.8.0/b2sdk/v1/account_info.py000066400000000000000000000135351474454370000205150ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/account_info.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from abc import abstractmethod import inspect import logging import os from b2sdk import v2 from b2sdk._internal.account_info.sqlite_account_info import DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE from b2sdk._internal.utils import limit_trace_arguments logger = logging.getLogger(__name__) # Retain legacy get_minimum_part_size and facilitate for optional s3_api_url class OldAccountInfoMethods: REALM_URLS = v2.REALM_URLS @limit_trace_arguments( only=[ 'self', 'api_url', 'download_url', 'minimum_part_size', 'realm', 's3_api_url', ] ) def set_auth_data( self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, allowed=None, application_key_id=None, s3_api_url=None, ): if 's3_api_url' in inspect.getfullargspec(self._set_auth_data).args: s3_kwargs = dict(s3_api_url=s3_api_url) else: s3_kwargs = {} if allowed is None: allowed = self.DEFAULT_ALLOWED assert self.allowed_is_valid(allowed) self._set_auth_data( account_id=account_id, auth_token=auth_token, api_url=api_url, download_url=download_url, minimum_part_size=minimum_part_size, application_key=application_key, realm=realm, allowed=allowed, application_key_id=application_key_id, **s3_kwargs, ) # translate legacy "minimum_part_size" to new style "recommended_part_size" class MinimumPartSizeTranslator: def _set_auth_data( self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, s3_api_url=None, allowed=None, application_key_id=None, ): if 's3_api_url' in inspect.getfullargspec(super()._set_auth_data).args: s3_kwargs = dict(s3_api_url=s3_api_url) else: s3_kwargs = {} return super()._set_auth_data( account_id=account_id, auth_token=auth_token, api_url=api_url, download_url=download_url, recommended_part_size=minimum_part_size, absolute_minimum_part_size=DEFAULT_ABSOLUTE_MINIMUM_PART_SIZE, application_key=application_key, realm=realm, allowed=allowed, application_key_id=application_key_id, **s3_kwargs, ) def get_minimum_part_size(self): return self.get_recommended_part_size() class AbstractAccountInfo(OldAccountInfoMethods, v2.AbstractAccountInfo): def get_s3_api_url(self): """ Return s3_api_url or raises MissingAccountData exception. :rtype: str """ # Removed @abstractmethod decorators def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> str | None: """ Look up the bucket name for the given bucket id. """ # Removed @abstractmethod decorator def get_recommended_part_size(self): """ Return the recommended number of bytes in a part of a large file. :return: number of bytes :rtype: int """ # Removed @abstractmethod decorator def get_absolute_minimum_part_size(self): """ Return the absolute minimum number of bytes in a part of a large file. :return: number of bytes :rtype: int """ # Removed @abstractmethod decorator @abstractmethod def get_minimum_part_size(self): """ Return the minimum number of bytes in a part of a large file. :return: number of bytes :rtype: int """ # This stays abstract in v1 @abstractmethod def _set_auth_data( self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, realm, s3_api_url, allowed, application_key_id, ): """ Actually store the auth data. Can assume that 'allowed' is present and valid. All of the information returned by ``b2_authorize_account`` is saved, because all of it is needed at some point. """ # Keep the old signature class InMemoryAccountInfo(MinimumPartSizeTranslator, OldAccountInfoMethods, v2.InMemoryAccountInfo): pass class UrlPoolAccountInfo(OldAccountInfoMethods, v2.UrlPoolAccountInfo): pass class SqliteAccountInfo(MinimumPartSizeTranslator, OldAccountInfoMethods, v2.SqliteAccountInfo): def __init__(self, file_name=None, last_upgrade_to_run=None): """ If ``file_name`` argument is empty or ``None``, path from ``B2_ACCOUNT_INFO`` environment variable is used. If that is not available, a default of ``~/.b2_account_info`` is used. :param str file_name: The sqlite file to use; overrides the default. :param int last_upgrade_to_run: For testing only, override the auto-update on the db. """ # use legacy env var resolution, XDG not supported file_name = file_name or os.environ.get( v2.B2_ACCOUNT_INFO_ENV_VAR, v2.B2_ACCOUNT_INFO_DEFAULT_FILE ) super().__init__(file_name=file_name, last_upgrade_to_run=last_upgrade_to_run) class StubAccountInfo(MinimumPartSizeTranslator, OldAccountInfoMethods, v2.StubAccountInfo): REALM_URLS = {'production': 'http://production.example.com'} b2-sdk-python-2.8.0/b2sdk/v1/api.py000066400000000000000000000203501474454370000166100ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from typing import Any, overload from .download_dest import AbstractDownloadDestination from b2sdk import v2 from b2sdk._internal.api import Services from .account_info import AbstractAccountInfo from .bucket import Bucket, BucketFactory, download_file_and_return_info_dict from .cache import AbstractCache from .file_version import ( FileVersionInfo, FileVersionInfoFactory, file_version_info_from_id_and_name, ) from .session import B2Session # override to use legacy no-request method of creating a bucket from bucket_id and retain `check_bucket_restrictions` # public API method # and to use v1.Bucket # and to retain cancel_large_file return type # and to retain old style download_file_by_id signature (allowing for the new one as well) and exception # and to retain old style get_file_info return type # and to accept old-style raw_api argument # and to retain old style create_key, delete_key and list_keys interfaces and behaviour class B2Api(v2.B2Api): SESSION_CLASS = staticmethod(B2Session) BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) BUCKET_CLASS = staticmethod(Bucket) FILE_VERSION_FACTORY_CLASS = staticmethod(FileVersionInfoFactory) def __init__( self, account_info: AbstractAccountInfo | None = None, cache: AbstractCache | None = None, raw_api: v2.B2RawHTTPApi = None, max_upload_workers: int = 10, max_copy_workers: int = 10, api_config: v2.B2HttpApiConfig | None = None, ): """ Initialize the API using the given account info. :param account_info: To learn more about Account Info objects, see here :class:`~b2sdk.v1.SqliteAccountInfo` :param cache: It is used by B2Api to cache the mapping between bucket name and bucket ids. default is :class:`~b2sdk._internal.cache.DummyCache` :param max_upload_workers: a number of upload threads :param max_copy_workers: a number of copy threads :param raw_api: :param api_config: """ self.session = self.SESSION_CLASS( account_info=account_info, cache=cache, raw_api=raw_api, api_config=api_config, ) self.file_version_factory = self.FILE_VERSION_FACTORY_CLASS(self) self.download_version_factory = self.DOWNLOAD_VERSION_FACTORY_CLASS(self) self.services = Services( self, max_upload_workers=max_upload_workers, max_copy_workers=max_copy_workers, ) def get_file_info(self, file_id: str) -> dict[str, Any]: """ Gets info about file version. :param str file_id: the id of the file. """ return self.session.get_file_info_by_id(file_id) def get_bucket_by_id(self, bucket_id): """ Return a bucket object with a given ID. Unlike ``get_bucket_by_name``, this method does not need to make any API calls. :param str bucket_id: a bucket ID :return: a Bucket object :rtype: b2sdk.v1.Bucket """ return self.BUCKET_CLASS(self, bucket_id) def check_bucket_restrictions(self, bucket_name): """ Check to see if the allowed field from authorize-account has a bucket restriction. If it does, checks if the bucket_name for a given api call matches that. If not, it raises a :py:exc:`b2sdk.v1.exception.RestrictedBucket` error. :param str bucket_name: a bucket name :raises b2sdk.v1.exception.RestrictedBucket: if the account is not allowed to use this bucket """ self.check_bucket_name_restrictions(bucket_name) def cancel_large_file(self, file_id: str) -> FileVersionInfo: file_id_and_name = super().cancel_large_file(file_id) return file_version_info_from_id_and_name(file_id_and_name, self) @overload def download_file_by_id( self, file_id: str, download_dest: AbstractDownloadDestination, progress_listener: v2.AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: v2.EncryptionSetting | None = None, ) -> dict: ... @overload def download_file_by_id( self, file_id: str, progress_listener: v2.AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: v2.EncryptionSetting | None = None, ) -> v2.DownloadedFile: ... def download_file_by_id( self, file_id: str, download_dest: AbstractDownloadDestination | None = None, progress_listener: v2.AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: v2.EncryptionSetting | None = None, ): """ Download a file with the given ID. :param file_id: a file ID :param download_dest: an instance of the one of the following classes: \ :class:`~b2sdk.v1.DownloadDestLocalFile`,\ :class:`~b2sdk.v1.DownloadDestBytes`,\ :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` :param progress_listener: an instance of the one of the following classes: \ :class:`~b2sdk.v1.PartProgressReporter`,\ :class:`~b2sdk.v1.TqdmProgressListener`,\ :class:`~b2sdk.v1.SimpleProgressListener`,\ :class:`~b2sdk.v1.DoNothingProgressListener`,\ :class:`~b2sdk.v1.ProgressListenerForTest`,\ :class:`~b2sdk.v1.SyncFileReporter`,\ or any sub class of :class:`~b2sdk.v1.AbstractProgressListener` :param range_: a list of two integers, the first one is a start\ position, and the second one is the end position in the file :param encryption: encryption settings (``None`` if unknown) """ downloaded_file = super().download_file_by_id( file_id=file_id, progress_listener=progress_listener, range_=range_, encryption=encryption, ) if download_dest is not None: try: return download_file_and_return_info_dict(downloaded_file, download_dest, range_) except ValueError as ex: if ex.args == ('no strategy suitable for download was found!',): raise AssertionError('no strategy suitable for download was found!') raise else: return downloaded_file def list_keys(self, start_application_key_id=None) -> dict: """ List application keys. Perform a single request and return at most ``self.DEFAULT_LIST_KEY_COUNT`` keys, as well as the value to supply to the next call as ``start_application_key_id``, if not all keys were retrieved. :param start_application_key_id: an :term:`application key ID` to start from or ``None`` to start from the beginning """ account_id = self.account_info.get_account_id() return self.session.list_keys( account_id, max_key_count=self.DEFAULT_LIST_KEY_COUNT, start_application_key_id=start_application_key_id, ) def create_key( self, capabilities: list[str], key_name: str, valid_duration_seconds: int | None = None, bucket_id: str | None = None, name_prefix: str | None = None, ): return ( super() .create_key( capabilities=capabilities, key_name=key_name, valid_duration_seconds=valid_duration_seconds, bucket_id=bucket_id, name_prefix=name_prefix, ) .as_dict() ) def delete_key(self, application_key_id): return super().delete_key_by_id(application_key_id).as_dict() def get_key(self, key_id: str) -> dict | None: keys = self.list_keys(start_application_key_id=key_id)['keys'] return next((key for key in keys if key['applicationKeyId'] == key_id), None) b2-sdk-python-2.8.0/b2sdk/v1/b2http.py000066400000000000000000000036101474454370000172420ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/b2http.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import requests from b2sdk import v2 # Overridden to retain old-style __init__ signature class B2Http(v2.B2Http): """ A wrapper for the requests module. Provides the operations needed to access B2, and handles retrying when the returned status is 503 Service Unavailable, 429 Too Many Requests, etc. The operations supported are: - post_json_return_json - post_content_return_json - get_content The methods that return JSON either return a Python dict or raise a subclass of B2Error. They can be used like this: .. code-block:: python try: response_dict = b2_http.post_json_return_json(url, headers, params) ... except B2Error as e: ... """ # timeout for HTTP GET/POST requests TIMEOUT = 1200 # 20 minutes as server-side copy can take time def __init__(self, requests_module=None, install_clock_skew_hook=True, user_agent_append=None): """ Initialize with a reference to the requests module, which makes it easy to mock for testing. :param requests_module: a reference to requests module :param bool install_clock_skew_hook: if True, install a clock skew hook :param str user_agent_append: if provided, the string will be appended to the User-Agent """ super().__init__( v2.B2HttpApiConfig( http_session_factory=(requests_module or requests).Session, install_clock_skew_hook=install_clock_skew_hook, user_agent_append=user_agent_append, ) ) b2-sdk-python-2.8.0/b2sdk/v1/bucket.py000066400000000000000000000326041474454370000173210ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/bucket.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from contextlib import suppress from typing import overload from .download_dest import AbstractDownloadDestination from .file_metadata import FileMetadata from .file_version import ( FileVersionInfo, FileVersionInfoFactory, file_version_info_from_download_version, ) from b2sdk import v2 from b2sdk._internal.utils import validate_b2_file_name from b2sdk._internal.raw_api import LifecycleRule # Overridden to retain the obsolete copy_file and start_large_file methods # and to retain old style FILE_VERSION_FACTORY attribute # and to retain old style download_file_by_name signature # and to retain old style download_file_by_id signature (allowing for the new one as well) # and to retain old style get_file_info_by_name return type # and to to adjust to old style B2Api.get_file_info return type # and to retain old style update return type class Bucket(v2.Bucket): FILE_VERSION_FACTORY = staticmethod(FileVersionInfoFactory) def copy_file( self, file_id, new_file_name, bytes_range=None, metadata_directive=None, content_type=None, file_info=None, destination_encryption: v2.EncryptionSetting | None = None, source_encryption: v2.EncryptionSetting | None = None, file_retention: v2.FileRetentionSetting | None = None, legal_hold: v2.LegalHold | None = None, cache_control: str | None = None, ): """ Creates a new file in this bucket by (server-side) copying from an existing file. :param str file_id: file ID of existing file :param str new_file_name: file name of the new file :param tuple[int,int],None bytes_range: start and end offsets (**inclusive!**), default is the entire file :param b2sdk.v1.MetadataDirectiveMode,None metadata_directive: default is :py:attr:`b2sdk.v1.MetadataDirectiveMode.COPY` :param str,None content_type: content_type for the new file if metadata_directive is set to :py:attr:`b2sdk.v1.MetadataDirectiveMode.REPLACE`, default will copy the content_type of old file :param dict,None file_info: file_info for the new file if metadata_directive is set to :py:attr:`b2sdk.v1.MetadataDirectiveMode.REPLACE`, default will copy the file_info of old file :param b2sdk.v1.EncryptionSetting destination_encryption: encryption settings for the destination (``None`` if unknown) :param b2sdk.v1.EncryptionSetting source_encryption: encryption settings for the source (``None`` if unknown) :param b2sdk.v1.FileRetentionSetting file_retention: retention setting for the new file :param bool legal_hold: legalHold setting for the new file :param str cache_control: cache control setting for the new file. Syntax based on the section 14.9 of RC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. """ file_info = file_info or {} if cache_control is not None: file_info['b2-cache-control'] = cache_control return self.api.session.copy_file( file_id, new_file_name, bytes_range, metadata_directive, content_type, file_info, self.id_, destination_server_side_encryption=destination_encryption, source_server_side_encryption=source_encryption, file_retention=file_retention, legal_hold=legal_hold, ) def start_large_file( self, file_name, content_type=None, file_info=None, file_retention: v2.FileRetentionSetting | None = None, legal_hold: v2.LegalHold | None = None, cache_control: str | None = None, ): """ Start a large file transfer. :param str file_name: a file name :param str,None content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param dict,None file_info: a file info to store with the file or ``None`` to not store anything :param b2sdk.v1.FileRetentionSetting file_retention: retention setting for the new file :param bool legal_hold: legalHold setting for the new file :param str,None cache_control: an optional cache control setting. Syntax based on the section 14.9 of RFC 2616. Example string value: 'public, max-age=86400, s-maxage=3600, no-transform'. """ file_info = file_info or {} if cache_control is not None: file_info['b2-cache-control'] = cache_control validate_b2_file_name(file_name) return self.api.services.large_file.start_large_file( self.id_, file_name, content_type=content_type, file_info=file_info, file_retention=file_retention, legal_hold=legal_hold, ) def download_file_by_name( self, file_name: str, download_dest: AbstractDownloadDestination, progress_listener: v2.AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: v2.EncryptionSetting | None = None, ): """ Download a file by name. .. seealso:: :ref:`Synchronizer `, a *high-performance* utility that synchronizes a local folder with a Bucket. :param str file_name: a file name :param download_dest: an instance of the one of the following classes: \ :class:`~b2sdk.v1.DownloadDestLocalFile`,\ :class:`~b2sdk.v1.DownloadDestBytes`,\ :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` :param progress_listener: a progress listener object to use, or ``None`` to not track progress :param range_: two integer values, start and end offsets :param encryption: encryption settings (``None`` if unknown) """ downloaded_file = super().download_file_by_name( file_name=file_name, progress_listener=progress_listener, range_=range_, encryption=encryption, ) try: return download_file_and_return_info_dict(downloaded_file, download_dest, range_) except ValueError as ex: if ex.args == ('no strategy suitable for download was found!',): raise AssertionError('no strategy suitable for download was found!') raise @overload def download_file_by_id( self, file_id: str, download_dest: AbstractDownloadDestination = None, progress_listener: v2.AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: v2.EncryptionSetting | None = None, ) -> dict: ... @overload def download_file_by_id( self, file_id: str, progress_listener: v2.AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: v2.EncryptionSetting | None = None, ) -> v2.DownloadedFile: ... def download_file_by_id( self, file_id: str, download_dest: AbstractDownloadDestination | None = None, progress_listener: v2.AbstractProgressListener | None = None, range_: tuple[int, int] | None = None, encryption: v2.EncryptionSetting | None = None, ): """ Download a file by ID. .. note:: download_file_by_id actually belongs in :py:class:`b2sdk.v1.B2Api`, not in :py:class:`b2sdk.v1.Bucket`; we just provide a convenient redirect here :param file_id: a file ID :param download_dest: an instance of the one of the following classes: \ :class:`~b2sdk.v1.DownloadDestLocalFile`,\ :class:`~b2sdk.v1.DownloadDestBytes`,\ :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` :param progress_listener: a progress listener object to use, or ``None`` to not report progress :param range_: two integer values, start and end offsets :param encryption: encryption settings (``None`` if unknown) """ return self.api.download_file_by_id( file_id, download_dest, progress_listener, range_=range_, encryption=encryption, ) def get_file_info_by_name(self, file_name: str) -> FileVersionInfo: return file_version_info_from_download_version(super().get_file_info_by_name(file_name)) def get_file_info_by_id(self, file_id: str) -> FileVersionInfo: """ Gets a file version's by ID. :param str file_id: the id of the file. """ return self.api.file_version_factory.from_api_response(self.api.get_file_info(file_id)) def update( self, bucket_type: str | None = None, bucket_info: dict | None = None, cors_rules: dict | None = None, lifecycle_rules: list[LifecycleRule] | None = None, if_revision_is: int | None = None, default_server_side_encryption: v2.EncryptionSetting | None = None, default_retention: v2.BucketRetentionSetting | None = None, is_file_lock_enabled: bool | None = None, **kwargs, ): """ Update various bucket parameters. :param bucket_type: a bucket type, e.g. ``allPrivate`` or ``allPublic`` :param bucket_info: an info to store with a bucket :param cors_rules: CORS rules to store with a bucket :param lifecycle_rules: lifecycle rules of the bucket :param if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is* :param default_server_side_encryption: default server side encryption settings (``None`` if unknown) :param default_retention: bucket default retention setting :param bool is_file_lock_enabled: specifies whether bucket should get File Lock-enabled """ # allow common tests to execute without hitting attributeerror with suppress(KeyError): del kwargs['replication'] self.replication = None assert ( not kwargs ) # after we get rid of everything we don't support in this apiver, this should be empty account_id = self.api.account_info.get_account_id() return self.api.session.update_bucket( account_id, self.id_, bucket_type=bucket_type, bucket_info=bucket_info, cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, default_retention=default_retention, is_file_lock_enabled=is_file_lock_enabled, ) def ls( self, folder_to_list: str = '', show_versions: bool = False, recursive: bool = False, fetch_count: int | None = 10000, **kwargs, ): """ Pretend that folders exist and yields the information about the files in a folder. B2 has a flat namespace for the files in a bucket, but there is a convention of using "/" as if there were folders. This method searches through the flat namespace to find the files and "folders" that live within a given folder. When the `recursive` flag is set, lists all of the files in the given folder, and all of its sub-folders. :param folder_to_list: the name of the folder to list; must not start with "/". Empty string means top-level folder :param show_versions: when ``True`` returns info about all versions of a file, when ``False``, just returns info about the most recent versions :param recursive: if ``True``, list folders recursively :param fetch_count: how many entries to return or ``None`` to use the default. Acceptable values: 1 - 10000 :rtype: generator[tuple[b2sdk.v1.FileVersionInfo, str]] :returns: generator of (file_version, folder_name) tuples .. note:: In case of `recursive=True`, folder_name is not returned. """ return super().ls(folder_to_list, not show_versions, recursive, fetch_count, **kwargs) def download_file_and_return_info_dict( downloaded_file: v2.DownloadedFile, download_dest: AbstractDownloadDestination, range_: tuple[int, int] | None, ): with download_dest.make_file_context( file_id=downloaded_file.download_version.id_, file_name=downloaded_file.download_version.file_name, content_length=downloaded_file.download_version.size, content_type=downloaded_file.download_version.content_type, content_sha1=downloaded_file.download_version.content_sha1, file_info=downloaded_file.download_version.file_info, mod_time_millis=downloaded_file.download_version.mod_time_millis, range_=range_, ) as file: downloaded_file.save(file) return FileMetadata.from_download_version(downloaded_file.download_version).as_info_dict() class BucketFactory(v2.BucketFactory): BUCKET_CLASS = staticmethod(Bucket) b2-sdk-python-2.8.0/b2sdk/v1/cache.py000066400000000000000000000010221474454370000170750ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/cache.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import v2 class AbstractCache(v2.AbstractCache): def get_bucket_name_or_none_from_bucket_id(self, bucket_id: str) -> str | None: return None # Removed @abstractmethod decorator b2-sdk-python-2.8.0/b2sdk/v1/download_dest.py000066400000000000000000000155521474454370000206750ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/download_dest.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import io import os from abc import abstractmethod from contextlib import contextmanager from b2sdk._internal.stream.progress import WritingStreamWithProgress from b2sdk._internal.utils import B2TraceMetaAbstract, limit_trace_arguments, set_file_mtime class AbstractDownloadDestination(metaclass=B2TraceMetaAbstract): """ Interface to a destination for a downloaded file. """ @abstractmethod @limit_trace_arguments( skip=[ 'content_sha1', ] ) def make_file_context( self, file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_=None, ): """ Return a context manager that yields a binary file-like object to use for writing the contents of the file. :param str file_id: the B2 file ID from the headers :param str file_name: the B2 file name from the headers :param str content_length: the content length :param str content_type: the content type from the headers :param str content_sha1: the content sha1 from the headers (or ``"none"`` for large files) :param dict file_info: the user file info from the headers :param int mod_time_millis: the desired file modification date in ms since 1970-01-01 :param None,tuple[int,int] range_: starting and ending offsets of the received file contents. Usually ``None``, which means that the whole file is downloaded. :return: None """ class DownloadDestLocalFile(AbstractDownloadDestination): """ Store a downloaded file into a local file and sets its modification time. """ MODE = 'wb+' def __init__(self, local_file_path): self.local_file_path = local_file_path def make_file_context( self, file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_=None, ): self.file_id = file_id self.file_name = file_name self.content_length = content_length self.content_type = content_type self.content_sha1 = content_sha1 self.file_info = file_info self.range_ = range_ return self.write_to_local_file_context(mod_time_millis) @contextmanager def write_to_local_file_context(self, mod_time_millis): completed = False try: # Open the file and let the caller write it. with open(self.local_file_path, self.MODE) as f: yield f set_file_mtime(self.local_file_path, mod_time_millis) # Set the flag that means to leave the downloaded file on disk. completed = True finally: # This is a best-effort attempt to clean up files that # failed to download, so we don't leave partial files # sitting on disk. if not completed: os.unlink(self.local_file_path) class PreSeekedDownloadDest(DownloadDestLocalFile): """ Store a downloaded file into a local file and sets its modification time. Does not truncate the target file, seeks to a given offset just after opening a descriptor. """ MODE = 'rb+' def __init__(self, local_file_path, seek_target): self._seek_target = seek_target super().__init__(local_file_path) @contextmanager def write_to_local_file_context(self, *args, **kwargs): with super().write_to_local_file_context(*args, **kwargs) as f: f.seek(self._seek_target) yield f class DownloadDestBytes(AbstractDownloadDestination): """ Store a downloaded file into bytes in memory. """ def __init__(self): self.bytes_written = None def make_file_context( self, file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_=None, ): self.file_id = file_id self.file_name = file_name self.content_length = content_length self.content_type = content_type self.content_sha1 = content_sha1 self.file_info = file_info self.mod_time_millis = mod_time_millis self.range_ = range_ return self.capture_bytes_context() @contextmanager def capture_bytes_context(self): """ Remember the bytes written in self.bytes_written. """ # Make a place to store the data written bytes_io = io.BytesIO() # Let the caller write it yield bytes_io # Capture the result. The BytesIO object won't let you grab # the data after it's closed self.bytes_written = bytes_io.getvalue() bytes_io.close() def get_bytes_written(self): if self.bytes_written is None: raise Exception('data not written yet') return self.bytes_written class DownloadDestProgressWrapper(AbstractDownloadDestination): """ Wrap a DownloadDestination and report progress to a ProgressListener. """ def __init__(self, download_dest, progress_listener): self.download_dest = download_dest self.progress_listener = progress_listener def make_file_context( self, file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_=None, ): return self.write_file_and_report_progress_context( file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_, ) @contextmanager def write_file_and_report_progress_context( self, file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_, ): with self.download_dest.make_file_context( file_id, file_name, content_length, content_type, content_sha1, file_info, mod_time_millis, range_, ) as file_: total_bytes = content_length if range_ is not None: total_bytes = range_[1] - range_[0] + 1 self.progress_listener.set_total_bytes(total_bytes) with self.progress_listener: yield WritingStreamWithProgress(file_, self.progress_listener) b2-sdk-python-2.8.0/b2sdk/v1/exception.py000066400000000000000000000027101474454370000200350ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk.v2.exception import * # noqa v2DestFileNewer = DestFileNewer # This exception class is deprecated and should not be used in new designs class CommandError(B2Error): """ b2 command error (user caused). Accepts exactly one argument: message. We expect users of shell scripts will parse our ``__str__`` output. """ def __init__(self, message): super().__init__() self.message = message def __str__(self): return self.message class DestFileNewer(v2DestFileNewer): def __init__(self, dest_file, source_file, dest_prefix, source_prefix): super(v2DestFileNewer, self).__init__() self.dest_file = dest_file self.source_file = source_file self.dest_prefix = dest_prefix self.source_prefix = source_prefix def __str__(self): return f'source file is older than destination: {self.source_prefix}{self.source_file.name} with a time of {self.source_file.latest_version().mod_time} cannot be synced to {self.dest_prefix}{self.dest_file.name} with a time of {self.dest_file.latest_version().mod_time}, unless a valid newer_file_mode is provided' b2-sdk-python-2.8.0/b2sdk/v1/file_metadata.py000066400000000000000000000042541474454370000206230ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/file_metadata.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import b2sdk.v2 as v2 class FileMetadata: """ Hold information about a file which is being downloaded. """ UNVERIFIED_CHECKSUM_PREFIX = 'unverified:' def __init__( self, file_id, file_name, content_type, content_length, content_sha1, file_info, ): self.file_id = file_id self.file_name = file_name self.content_type = content_type self.content_length = content_length self.content_sha1, self.content_sha1_verified = self._decode_content_sha1(content_sha1) self.file_info = file_info def as_info_dict(self): return { 'fileId': self.file_id, 'fileName': self.file_name, 'contentType': self.content_type, 'contentLength': self.content_length, 'contentSha1': self._encode_content_sha1(self.content_sha1, self.content_sha1_verified), 'fileInfo': self.file_info, } @classmethod def _decode_content_sha1(cls, content_sha1): if content_sha1.startswith(cls.UNVERIFIED_CHECKSUM_PREFIX): return content_sha1[len(cls.UNVERIFIED_CHECKSUM_PREFIX) :], False return content_sha1, True @classmethod def _encode_content_sha1(cls, content_sha1, content_sha1_verified): if not content_sha1_verified: return f'{cls.UNVERIFIED_CHECKSUM_PREFIX}{content_sha1}' return content_sha1 @classmethod def from_download_version(cls, download_version: v2.DownloadVersion): return cls( file_id=download_version.id_, file_name=download_version.file_name, content_type=download_version.content_type, content_length=download_version.content_length, content_sha1=download_version.content_sha1, file_info=download_version.file_info, ) b2-sdk-python-2.8.0/b2sdk/v1/file_version.py000066400000000000000000000156221474454370000205310ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/file_version.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from contextlib import suppress import datetime import functools from b2sdk import v2 from . import api as v1api # Override to retain legacy class name, __init__ signature, slots # and old formatting methods # and to omit 'api' property when doing __eq__ and __repr__ # and to make get_fresh_state return proper objects, even though v1.B2Api.get_file_info returns dicts class FileVersionInfo(v2.FileVersion): __slots__ = ['_api'] LS_ENTRY_TEMPLATE = ( '%83s %6s %10s %8s %9d %s' # order is file_id, action, date, time, size, name ) def __init__( self, id_, file_name, size, content_type, content_sha1, file_info, upload_timestamp, action, account_id: str | None = None, bucket_id: str | None = None, content_md5=None, server_side_encryption: v2.EncryptionSetting | None = None, file_retention: v2.FileRetentionSetting | None = None, legal_hold: v2.LegalHold | None = None, api: v1api.B2Api | None = None, cache_control: str | None = None, **kwargs, ): self.id_ = id_ self.file_name = file_name self.size = size and int(size) self.content_type = content_type self.content_sha1, self.content_sha1_verified = self._decode_content_sha1(content_sha1) self.account_id = account_id self.bucket_id = bucket_id self.content_md5 = content_md5 self.file_info = file_info or {} self.upload_timestamp = upload_timestamp self.action = action self.server_side_encryption = server_side_encryption self.legal_hold = legal_hold self.file_retention = file_retention self._api = api self.cache_control = cache_control if self.cache_control is None: self.cache_control = (file_info or {}).get('b2-cache-control') # allow common tests to execute without hitting attributeerror with suppress(KeyError): del kwargs['replication_status'] self.replication_status = None assert ( not kwargs ) # after we get rid of everything we don't support in this apiver, this should be empty if v2.SRC_LAST_MODIFIED_MILLIS in self.file_info: self.mod_time_millis = int(self.file_info[v2.SRC_LAST_MODIFIED_MILLIS]) else: self.mod_time_millis = self.upload_timestamp @property def api(self): if self._api is None: raise ValueError('"api" not set') return self._api def _all_slots(self): all_slots = super()._all_slots() all_slots.remove('api') return all_slots def format_ls_entry(self): dt = datetime.datetime.utcfromtimestamp(self.upload_timestamp / 1000) date_str = dt.strftime('%Y-%m-%d') time_str = dt.strftime('%H:%M:%S') size = self.size or 0 # required if self.action == 'hide' return self.LS_ENTRY_TEMPLATE % ( self.id_, self.action, date_str, time_str, size, self.file_name, ) @classmethod def format_folder_ls_entry(cls, name): return cls.LS_ENTRY_TEMPLATE % ('-', '-', '-', '-', 0, name) def get_fresh_state(self) -> FileVersionInfo: """ Fetch all the information about this file version and return a new FileVersion object. This method does NOT change the object it is called on. """ return self.api.file_version_factory.from_api_response(self.api.get_file_info(self.id_)) def file_version_info_from_new_file_version(file_version: v2.FileVersion) -> FileVersionInfo: return FileVersionInfo( **{ att_name: getattr(file_version, att_name) for att_name in [ 'id_', 'file_name', 'size', 'content_type', 'content_sha1', 'file_info', 'upload_timestamp', 'action', 'content_md5', 'server_side_encryption', 'legal_hold', 'file_retention', 'cache_control', 'api', ] } ) def translate_single_file_version(func): @functools.wraps(func) def inner(*a, **kw): return file_version_info_from_new_file_version(func(*a, **kw)) return inner # override to return old style FileVersionInfo class FileVersionInfoFactory(v2.FileVersionFactory): from_api_response = translate_single_file_version(v2.FileVersionFactory.from_api_response) def from_response_headers(self, headers): file_info = v2.DownloadVersionFactory.file_info_from_headers(headers) return FileVersionInfo( api=self.api, id_=headers['x-bz-file-id'], file_name=headers['x-bz-file-name'], size=int(headers['content-length']), content_type=headers['content-type'], content_sha1=headers['x-bz-content-sha1'], file_info=file_info, upload_timestamp=int(headers['x-bz-upload-timestamp']), action='upload', content_md5=None, server_side_encryption=v2.EncryptionSettingFactory.from_response_headers(headers), file_retention=v2.FileRetentionSetting.from_response_headers(headers), legal_hold=v2.LegalHold.from_response_headers(headers), cache_control=headers['Cache-control'], ) def file_version_info_from_id_and_name(file_id_and_name: v2.FileIdAndName, api: v1api.B2Api): return FileVersionInfo( id_=file_id_and_name.file_id, file_name=file_id_and_name.file_name, size=0, content_type='unknown', content_sha1='none', file_info={}, upload_timestamp=0, action='cancel', api=api, ) def file_version_info_from_download_version(download_version: v2.DownloadVersion): return FileVersionInfo( id_=download_version.id_, file_name=download_version.file_name, size=download_version.size, content_type=download_version.content_type, content_sha1=download_version.content_sha1, file_info=download_version.file_info, upload_timestamp=download_version.upload_timestamp, action='upload', content_md5=None, server_side_encryption=download_version.server_side_encryption, file_retention=download_version.file_retention, legal_hold=download_version.legal_hold, cache_control=download_version.cache_control, api=download_version.api, ) b2-sdk-python-2.8.0/b2sdk/v1/replication/000077500000000000000000000000001474454370000177765ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/v1/replication/__init__.py000066400000000000000000000005161474454370000221110ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/replication/__init__.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/b2sdk/v1/replication/monitoring.py000066400000000000000000000021731474454370000225400ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/replication/monitoring.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from dataclasses import dataclass import b2sdk.v2 as v2 from .. import Bucket from ..sync.folder import B2Folder @dataclass class ReplicationMonitor(v2.ReplicationMonitor): # when passing in v1 Bucket objects to ReplicationMonitor, # the latter should use v1 B2Folder to correctly use # v1 Bucket's interface B2_FOLDER_CLASS = B2Folder @property def destination_bucket(self) -> Bucket: destination_api = self.destination_api or self.source_api bucket_id = self.rule.destination_bucket_id # when using `destination_api.get_bucket_by_id(bucket_id)`, # v1 will instantiate the bucket without its name, but we need it, # so we use `list_buckets` to actually fetch bucket name return destination_api.list_buckets(bucket_id=bucket_id)[0] b2-sdk-python-2.8.0/b2sdk/v1/session.py000066400000000000000000000060051474454370000175230ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/session.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import v2 from b2sdk.v2.exception import InvalidArgument from .account_info import SqliteAccountInfo # Override to use legacy signature of account_info.set_auth_data, especially the minimum_part_size argument # and to accept old-style raw_api argument class B2Session(v2.B2Session): SQLITE_ACCOUNT_INFO_CLASS = staticmethod(SqliteAccountInfo) def __init__( self, account_info=None, cache=None, raw_api: v2.B2RawHTTPApi = None, api_config: v2.B2HttpApiConfig | None = None, ): if raw_api is not None and api_config is not None: raise InvalidArgument( 'raw_api,api_config', 'Provide at most one of: raw_api, api_config' ) if api_config is None: api_config = v2.DEFAULT_HTTP_API_CONFIG super().__init__(account_info=account_info, cache=cache, api_config=api_config) if raw_api is not None: self.raw_api = raw_api def authorize_account(self, realm, application_key_id, application_key): """ Perform account authorization. :param str realm: a realm to authorize account in (usually just "production") :param str application_key_id: :term:`application key ID` :param str application_key: user's :term:`application key` """ # Authorize realm_url = self.account_info.REALM_URLS.get(realm, realm) response = self.raw_api.authorize_account(realm_url, application_key_id, application_key) account_id = response['accountId'] storage_api_info = response['apiInfo']['storageApi'] # `allowed` object has been deprecated in the v3 of the API, but we still # construct it artificially to avoid changes in all the reliant parts. allowed = { 'bucketId': storage_api_info['bucketId'], 'bucketName': storage_api_info['bucketName'], 'capabilities': storage_api_info['capabilities'], 'namePrefix': storage_api_info['namePrefix'], } # Clear the cache if new account has been used if not self.account_info.is_same_account(account_id, realm): self.cache.clear() # Store the auth data self.account_info.set_auth_data( account_id=account_id, auth_token=response['authorizationToken'], api_url=storage_api_info['apiUrl'], download_url=storage_api_info['downloadUrl'], minimum_part_size=storage_api_info['recommendedPartSize'], application_key=application_key, realm=realm, s3_api_url=storage_api_info['s3ApiUrl'], allowed=allowed, application_key_id=application_key_id, ) b2-sdk-python-2.8.0/b2sdk/v1/sync/000077500000000000000000000000001474454370000164415ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/v1/sync/__init__.py000066400000000000000000000007711474454370000205570ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from .encryption_provider import * from .file import * from .folder import * from .folder_parser import * from .report import * from .scan_policies import * from .sync import * b2-sdk-python-2.8.0/b2sdk/v1/sync/encryption_provider.py000066400000000000000000000074351474454370000231300ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/encryption_provider.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import inspect from abc import abstractmethod from b2sdk import v2 from ..bucket import Bucket from ..file_version import FileVersionInfo # wrapper to translate new argument names to old ones class SyncEncryptionSettingsProviderWrapper(v2.AbstractSyncEncryptionSettingsProvider): def __init__(self, provider): self.provider = provider def __repr__(self): return f'{self.__class__.__name__}({self.provider})' def get_setting_for_upload( self, bucket: Bucket, b2_file_name: str, file_info: dict | None, length: int, ) -> v2.EncryptionSetting | None: return self.provider.get_setting_for_upload( bucket=bucket, b2_file_name=b2_file_name, file_info=file_info, length=length, ) def get_source_setting_for_copy( self, bucket: Bucket, source_file_version: v2.FileVersion, ) -> v2.EncryptionSetting | None: return self.provider.get_source_setting_for_copy( bucket=bucket, source_file_version_info=source_file_version ) def get_destination_setting_for_copy( self, bucket: Bucket, dest_b2_file_name: str, source_file_version: v2.FileVersion, target_file_info: dict | None = None, ) -> v2.EncryptionSetting | None: return self.provider.get_destination_setting_for_copy( bucket=bucket, dest_b2_file_name=dest_b2_file_name, source_file_version_info=source_file_version, target_file_info=target_file_info, ) def get_setting_for_download( self, bucket: Bucket, file_version: v2.FileVersion, ) -> v2.EncryptionSetting | None: return self.provider.get_setting_for_download( bucket=bucket, file_version_info=file_version, ) def wrap_if_necessary(provider): if 'file_version' in inspect.getfullargspec(provider.get_setting_for_download).args: return provider return SyncEncryptionSettingsProviderWrapper(provider) # Old signatures class AbstractSyncEncryptionSettingsProvider(v2.AbstractSyncEncryptionSettingsProvider): @abstractmethod def get_setting_for_upload( self, bucket: Bucket, b2_file_name: str, file_info: dict | None, length: int, ) -> v2.EncryptionSetting | None: """ Return an EncryptionSetting for uploading an object or None if server should decide. """ @abstractmethod def get_source_setting_for_copy( self, bucket: Bucket, source_file_version_info: FileVersionInfo, ) -> v2.EncryptionSetting | None: """ Return an EncryptionSetting for a source of copying an object or None if not required """ @abstractmethod def get_destination_setting_for_copy( self, bucket: Bucket, dest_b2_file_name: str, source_file_version_info: FileVersionInfo, target_file_info: dict | None = None, ) -> v2.EncryptionSetting | None: """ Return an EncryptionSetting for a destination for copying an object or None if server should decide """ @abstractmethod def get_setting_for_download( self, bucket: Bucket, file_version_info: FileVersionInfo, ) -> v2.EncryptionSetting | None: """ Return an EncryptionSetting for downloading an object from, or None if not required """ b2-sdk-python-2.8.0/b2sdk/v1/sync/file.py000066400000000000000000000072621474454370000177410ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/file.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk.v1 import FileVersionInfo from b2sdk._internal.http_constants import SRC_LAST_MODIFIED_MILLIS # This whole module is here to retain legacy classes so they can be used in retained legacy exception class File: """ Hold information about one file in a folder. The name is relative to the folder in all cases. Files that have multiple versions (which only happens in B2, not in local folders) include information about all of the versions, most recent first. """ __slots__ = ['name', 'versions'] def __init__(self, name, versions: list[FileVersion]): """ :param str name: a relative file name :param List[FileVersion] versions: a list of file versions """ self.name = name self.versions = versions def latest_version(self) -> FileVersion: """ Return the latest file version. """ return self.versions[0] def __repr__(self): return '{}({}, [{}])'.format( self.__class__.__name__, self.name, ', '.join(repr(v) for v in self.versions) ) class B2File(File): """ Hold information about one file in a folder in B2 cloud. """ __slots__ = ['name', 'versions'] def __init__(self, name, versions: list[FileVersion]): """ :param str name: a relative file name :param List[FileVersion] versions: a list of file versions """ super().__init__(name, versions) def latest_version(self) -> FileVersion: return super().latest_version() class FileVersion: """ Hold information about one version of a file. """ __slots__ = ['id_', 'name', 'mod_time', 'action', 'size'] def __init__(self, id_, file_name, mod_time, action, size): """ :param id_: the B2 file id, or the local full path name :type id_: str :param file_name: a relative file name :type file_name: str :param mod_time: modification time, in milliseconds, to avoid rounding issues with millisecond times from B2 :type mod_time: int :param action: "hide" or "upload" (never "start") :type action: str :param size: a file size :type size: int """ self.id_ = id_ self.name = file_name self.mod_time = mod_time self.action = action self.size = size def __repr__(self): return f'{self.__class__.__name__}({repr(self.id_)}, {repr(self.name)}, {repr(self.mod_time)}, {repr(self.action)})' class B2FileVersion(FileVersion): __slots__ = [ 'file_version_info' ] # in a typical use case there is a lot of these object in memory, hence __slots__ # and properties def __init__(self, file_version_info: FileVersionInfo): self.file_version_info = file_version_info @property def id_(self): return self.file_version_info.id_ @property def name(self): return self.file_version_info.file_name @property def mod_time(self): if SRC_LAST_MODIFIED_MILLIS in self.file_version_info.file_info: return int(self.file_version_info.file_info[SRC_LAST_MODIFIED_MILLIS]) return self.file_version_info.upload_timestamp @property def action(self): return self.file_version_info.action @property def size(self): return self.file_version_info.size b2-sdk-python-2.8.0/b2sdk/v1/sync/file_to_path_translator.py000066400000000000000000000053551474454370000237310ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/file_to_path_translator.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import v2 from .file import B2File, B2FileVersion, File, FileVersion # The goal is to create v1.File objects together with v1.FileVersion objects from v2.SyncPath objects def make_files_from_paths( dest_path: v2.AbstractSyncPath, source_path: v2.AbstractSyncPath, sync_type: str ) -> tuple[File, File]: assert sync_type in ('b2-to-b2', 'b2-to-local', 'local-to-b2') sync_type_split = sync_type.split('-') dest_type = sync_type_split[-1] dest_file = _path_translation_map[dest_type](dest_path) source_type = sync_type_split[0] source_file = _path_translation_map[source_type](source_path) return dest_file, source_file def _translate_b2_path_to_file(path: v2.B2SyncPath) -> B2File: versions = [B2FileVersion(version) for version in path.all_versions] return B2File(path.relative_path, versions) def _translate_local_path_to_file(path: v2.LocalSyncPath) -> File: version = FileVersion( id_=path.absolute_path, file_name=path.relative_path, mod_time=path.mod_time, action='upload', size=path.size, ) return File(path.relative_path, [version]) _path_translation_map = {'b2': _translate_b2_path_to_file, 'local': _translate_local_path_to_file} # The goal is to create v2.SyncPath objects from v1.File objects def make_paths_from_files( dest_file: File, source_file: File, sync_type: str ) -> tuple[v2.AbstractSyncPath, v2.AbstractSyncPath]: assert sync_type in ('b2-to-b2', 'b2-to-local', 'local-to-b2') sync_type_split = sync_type.split('-') dest_type = sync_type_split[-1] dest_path = _file_translation_map[dest_type](dest_file) source_type = sync_type_split[0] source_path = _file_translation_map[source_type](source_file) return dest_path, source_path def _translate_b2_file_to_path(file: B2File) -> v2.AbstractSyncPath: versions = [file_version.file_version_info for file_version in file.versions] return v2.B2SyncPath( relative_path=file.name, selected_version=versions[0], all_versions=versions ) def _translate_local_file_to_path(file: File) -> v2.AbstractSyncPath: return v2.LocalSyncPath( absolute_path=file.latest_version().id_, relative_path=file.name, mod_time=file.latest_version().mod_time, size=file.latest_version().size, ) _file_translation_map = {'b2': _translate_b2_file_to_path, 'local': _translate_local_file_to_path} b2-sdk-python-2.8.0/b2sdk/v1/sync/folder.py000066400000000000000000000044161474454370000202730ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/folder.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from abc import abstractmethod import functools from b2sdk import v2 from .scan_policies import DEFAULT_SCAN_MANAGER, wrap_if_necessary from .. import exception def translate_errors(func): @functools.wraps(func) def wrapper(*a, **kw): try: return func(*a, **kw) except exception.NotADirectory as ex: raise Exception(f'{ex.path} is not a directory') except exception.UnableToCreateDirectory as ex: raise Exception(f'unable to create directory {ex.path}') except exception.EmptyDirectory as ex: raise exception.CommandError( f'Directory {ex.path} is empty. Use --allowEmptySource to sync anyway.' ) return wrapper # Override to change "policies_manager" default argument class AbstractFolder(v2.AbstractFolder): @abstractmethod def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): pass # override to retain "policies_manager" default argument, # and wrap policies_manager class B2Folder(v2.B2Folder, AbstractFolder): def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): return super().all_files(reporter, wrap_if_necessary(policies_manager)) def get_file_versions(self): for file_version, _ in self.bucket.ls( self.folder_name, show_versions=True, recursive=True, ): yield file_version # override to retain "policies_manager" default argument, # translate nice errors to old style Exceptions and CommandError # and wrap policies_manager class LocalFolder(v2.LocalFolder, AbstractFolder): @translate_errors def ensure_present(self): return super().ensure_present() @translate_errors def ensure_non_empty(self): return super().ensure_non_empty() def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): return super().all_files(reporter, wrap_if_necessary(policies_manager)) b2-sdk-python-2.8.0/b2sdk/v1/sync/folder_parser.py000066400000000000000000000013421474454370000216420ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/folder_parser.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import v2 from .. import exception from .folder import LocalFolder, B2Folder # Override to use v1 version of "LocalFolder" and "B2Folder" and raise old style CommandError def parse_sync_folder(folder_name, api): try: return v2.parse_sync_folder(folder_name, api, LocalFolder, B2Folder) except exception.InvalidArgument as ex: raise exception.CommandError(ex.message) b2-sdk-python-2.8.0/b2sdk/v1/sync/report.py000066400000000000000000000015161474454370000203310ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/report.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import v2 # override to retain legacy methods class SyncReport(v2.SyncReport): @property def local_file_count(self): return self.total_count @local_file_count.setter def local_file_count(self, value): self.total_count = value @property def local_done(self): return self.total_done @local_done.setter def local_done(self, value): self.total_done = value update_local = v2.SyncReport.update_total end_local = v2.SyncReport.end_total b2-sdk-python-2.8.0/b2sdk/v1/sync/scan_policies.py000066400000000000000000000156311474454370000216340ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/scan_policies.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import re from typing import Iterable from .file import B2FileVersion from ..file_version import FileVersionInfo from .file_to_path_translator import _translate_local_path_to_file from b2sdk import v2 from b2sdk.v2 import exception as v2_exception # noqa # Override to retain old exceptions in __init__ # and to provide interface for new should_exclude_* methods class ScanPoliciesManager(v2.ScanPoliciesManager): """ Policy object used when scanning folders for syncing, used to decide which files to include in the list of files to be synced. Code that scans through files should at least use should_exclude_file() to decide whether each file should be included; it will check include/exclude patterns for file names, as well as patterns for excluding directories. Code that scans may optionally use should_exclude_directory() to test whether it can skip a directory completely and not bother listing the files and sub-directories in it. """ def __init__( self, exclude_dir_regexes: Iterable[str | re.Pattern] = tuple(), exclude_file_regexes: Iterable[str | re.Pattern] = tuple(), include_file_regexes: Iterable[str | re.Pattern] = tuple(), exclude_all_symlinks: bool = False, exclude_modified_before: int | None = None, exclude_modified_after: int | None = None, exclude_uploaded_before: int | None = None, exclude_uploaded_after: int | None = None, ): """ :param exclude_dir_regexes: regexes to exclude directories :param exclude_file_regexes: regexes to exclude files :param include_file_regexes: regexes to include files :param exclude_all_symlinks: if True, exclude all symlinks :param exclude_modified_before: optionally exclude file versions (both local and b2) modified before (in millis) :param exclude_modified_after: optionally exclude file versions (both local and b2) modified after (in millis) :param exclude_uploaded_before: optionally exclude b2 file versions uploaded before (in millis) :param exclude_uploaded_after: optionally exclude b2 file versions uploaded after (in millis) The regex matching priority for a given path is: 1) the path is always excluded if it's dir matches `exclude_dir_regexes`, if not then 2) the path is always included if it matches `include_file_regexes`, if not then 3) the path is excluded if it matches `exclude_file_regexes`, if not then 4) the path is included """ if include_file_regexes and not exclude_file_regexes: raise v2_exception.InvalidArgument( 'include_file_regexes', 'cannot be used without exclude_file_regexes at the same time', ) self._exclude_dir_set = v2.RegexSet(exclude_dir_regexes) self._exclude_file_because_of_dir_set = v2.RegexSet( map(v2.convert_dir_regex_to_dir_prefix_regex, exclude_dir_regexes) ) self._exclude_file_set = v2.RegexSet(exclude_file_regexes) self._include_file_set = v2.RegexSet(include_file_regexes) self.exclude_all_symlinks = exclude_all_symlinks self._include_mod_time_range = v2.IntegerRange( exclude_modified_before, exclude_modified_after ) with v2_exception.check_invalid_argument( 'exclude_uploaded_before,exclude_uploaded_after', '', ValueError ): self._include_upload_time_range = v2.IntegerRange( exclude_uploaded_before, exclude_uploaded_after ) def should_exclude_file(self, file_path): """ Given the full path of a file, decide if it should be excluded from the scan. :param file_path: the path of the file, relative to the root directory being scanned. :type: str :return: True if excluded. :rtype: bool """ if self._exclude_file_because_of_dir_set.matches(file_path): return True if self._include_file_set.matches(file_path): return False return self._exclude_file_set.matches(file_path) def should_exclude_file_version(self, file_version): """ Given the modification time of a file version, decide if it should be excluded from the scan. :param file_version: the file version object :type: b2sdk.v1.FileVersion :return: True if excluded. :rtype: bool """ return file_version.mod_time not in self._include_mod_time_range def should_exclude_directory(self, dir_path): """ Given the full path of a directory, decide if all of the files in it should be excluded from the scan. :param dir_path: the path of the directory, relative to the root directory being scanned. The path will never end in '/'. :type dir_path: str :return: True if excluded. """ return self._exclude_dir_set.matches(dir_path) class ScanPoliciesManagerWrapper(v2.ScanPoliciesManager): def __init__(self, scan_policies_manager: ScanPoliciesManager): self.scan_policies_manager = scan_policies_manager self.exclude_all_symlinks = scan_policies_manager.exclude_all_symlinks def __repr__(self): return f'{self.__class__.__name__}({self.scan_policies_manager})' def should_exclude_relative_path(self, relative_path: str): self.scan_policies_manager.should_exclude_file(relative_path) def should_exclude_local_path(self, local_path: v2.LocalSyncPath): if self.scan_policies_manager.should_exclude_file_version( _translate_local_path_to_file(local_path).latest_version() ): return True return self.scan_policies_manager.should_exclude_file(local_path.relative_path) def should_exclude_b2_file_version(self, file_version: FileVersionInfo, relative_path: str): if self.scan_policies_manager.should_exclude_file_version(B2FileVersion(file_version)): return True return self.scan_policies_manager.should_exclude_file(relative_path) def should_exclude_b2_directory(self, dir_path): return self.scan_policies_manager.should_exclude_directory(dir_path) def should_exclude_local_directory(self, dir_path): return self.scan_policies_manager.should_exclude_directory(dir_path) def wrap_if_necessary(scan_policies_manager): if hasattr(scan_policies_manager, 'should_exclude_file'): return ScanPoliciesManagerWrapper(scan_policies_manager) return scan_policies_manager DEFAULT_SCAN_MANAGER = ScanPoliciesManager() b2-sdk-python-2.8.0/b2sdk/v1/sync/sync.py000066400000000000000000000124361474454370000177750ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v1/sync/sync.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import v2 from b2sdk.v2 import exception as v2_exception from .file_to_path_translator import make_files_from_paths, make_paths_from_files from .scan_policies import DEFAULT_SCAN_MANAGER, wrap_if_necessary as scan_wrap_if_necessary from .encryption_provider import wrap_if_necessary as encryption_wrap_if_necessary from ..exception import DestFileNewer # Override to change "policies_manager" default argument def zip_folders(folder_a, folder_b, reporter, policies_manager=DEFAULT_SCAN_MANAGER): return v2.zip_folders( folder_a, folder_b, reporter, policies_manager=scan_wrap_if_necessary(policies_manager) ) # Override to change "policies_manager" default arguments # and to wrap encryption_settings_providers in argument name translators class Synchronizer(v2.Synchronizer): def __init__( self, max_workers, policies_manager=DEFAULT_SCAN_MANAGER, dry_run=False, allow_empty_source=False, newer_file_mode=v2.NewerFileSyncMode.RAISE_ERROR, keep_days_or_delete=v2.KeepOrDeleteMode.NO_DELETE, compare_version_mode=v2.CompareVersionMode.MODTIME, compare_threshold=None, keep_days=None, sync_policy_manager: v2.SyncPolicyManager = v2.POLICY_MANAGER, ): super().__init__( max_workers, scan_wrap_if_necessary(policies_manager), dry_run, allow_empty_source, newer_file_mode, keep_days_or_delete, compare_version_mode, compare_threshold, keep_days, sync_policy_manager, ) def make_folder_sync_actions( self, source_folder, dest_folder, now_millis, reporter, policies_manager=DEFAULT_SCAN_MANAGER, encryption_settings_provider=v2.SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): return super()._make_folder_sync_actions( source_folder, dest_folder, now_millis, reporter, scan_wrap_if_necessary(policies_manager), encryption_wrap_if_necessary(encryption_settings_provider), ) # override to retain a public method def make_file_sync_actions( self, sync_type, source_file, dest_file, source_folder, dest_folder, now_millis, encryption_settings_provider: v2.AbstractSyncEncryptionSettingsProvider = v2.SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ Yields the sequence of actions needed to sync the two files :param str sync_type: synchronization type :param b2sdk.v1.File source_file: source file object :param b2sdk.v1.File dest_file: destination file object :param b2sdk.v1.AbstractFolder source_folder: a source folder object :param b2sdk.v1.AbstractFolder dest_folder: a destination folder object :param int now_millis: current time in milliseconds :param b2sdk.v1.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ dest_path, source_path = make_paths_from_files(dest_file, source_file, sync_type) return self._make_file_sync_actions( sync_type, source_path, dest_path, source_folder, dest_folder, now_millis, encryption_wrap_if_necessary(encryption_settings_provider), ) # override to raise old style DestFileNewer exceptions def _make_file_sync_actions( self, sync_type, source_path, dest_path, source_folder, dest_folder, now_millis, encryption_settings_provider: v2.AbstractSyncEncryptionSettingsProvider = v2.SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER, ): """ Yields the sequence of actions needed to sync the two files :param str sync_type: synchronization type :param b2sdk.v1.AbstractSyncPath source_path: source file object :param b2sdk.v1.AbstractSyncPath dest_path: destination file object :param b2sdk.v1.AbstractFolder source_folder: a source folder object :param b2sdk.v1.AbstractFolder dest_folder: a destination folder object :param int now_millis: current time in milliseconds :param b2sdk.v1.AbstractSyncEncryptionSettingsProvider encryption_settings_provider: encryption setting provider """ try: yield from super()._make_file_sync_actions( sync_type, source_path, dest_path, source_folder, dest_folder, now_millis, encryption_wrap_if_necessary(encryption_settings_provider), ) except v2_exception.DestFileNewer as ex: dest_file, source_file = make_files_from_paths(ex.dest_path, ex.source_path, sync_type) raise DestFileNewer(dest_file, source_file, ex.dest_prefix, ex.source_prefix) b2-sdk-python-2.8.0/b2sdk/v2/000077500000000000000000000000001474454370000154665ustar00rootroot00000000000000b2-sdk-python-2.8.0/b2sdk/v2/__init__.py000066400000000000000000000025351474454370000176040ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._v3 import * # noqa from b2sdk._v3 import parse_folder as parse_sync_folder from b2sdk._v3 import AbstractPath as AbstractSyncPath from b2sdk._v3 import LocalPath as LocalSyncPath from b2sdk._internal.utils.escape import ( unprintable_to_hex, escape_control_chars, substitute_control_chars, ) from .account_info import AbstractAccountInfo from .api import B2Api from .b2http import B2Http from .bucket import Bucket, BucketFactory from .session import B2Session from .sync import B2SyncPath from .transfer import DownloadManager, UploadManager # utils from .version_utils import rename_argument, rename_function from .utils import TempDir # raw_simulator from .raw_simulator import BucketSimulator from .raw_simulator import RawSimulator # raw_api from .raw_api import AbstractRawApi from .raw_api import B2RawHTTPApi # file_version from .file_version import FileVersion from .file_version import FileVersionFactory # large_file from .large_file import LargeFileServices from .large_file import UnfinishedLargeFile b2-sdk-python-2.8.0/b2sdk/v2/_compat.py000066400000000000000000000007061474454370000174650ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/_compat.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal import version_utils _file_infos_rename = version_utils.rename_argument('file_infos', 'file_info', None, 'v3') b2-sdk-python-2.8.0/b2sdk/v2/account_info.py000066400000000000000000000007561474454370000205170ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/account_info.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import _v3 class AbstractAccountInfo(_v3.AbstractAccountInfo): def list_bucket_names_ids(self): return [] # Removed @abstractmethod decorator b2-sdk-python-2.8.0/b2sdk/v2/api.py000066400000000000000000000042741474454370000166200ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import _v3 as v3 from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound from .bucket import Bucket, BucketFactory from .exception import BucketIdNotFound from .session import B2Session from .transfer import DownloadManager, UploadManager from .file_version import FileVersionFactory from .large_file import LargeFileServices class Services(v3.Services): UPLOAD_MANAGER_CLASS = staticmethod(UploadManager) DOWNLOAD_MANAGER_CLASS = staticmethod(DownloadManager) LARGE_FILE_SERVICES_CLASS = staticmethod(LargeFileServices) # override to use legacy B2Session with legacy B2Http # and to raise old style BucketIdNotFound exception # and to use old style Bucket # and to use legacy authorize_account signature class B2Api(v3.B2Api): SESSION_CLASS = staticmethod(B2Session) BUCKET_CLASS = staticmethod(Bucket) BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) SERVICES_CLASS = staticmethod(Services) FILE_VERSION_FACTORY_CLASS = staticmethod(FileVersionFactory) # Legacy init in case something depends on max_workers defaults = 10 def __init__(self, *args, **kwargs): kwargs.setdefault('max_upload_workers', 10) kwargs.setdefault('max_copy_workers', 10) super().__init__(*args, **kwargs) def get_bucket_by_id(self, bucket_id: str) -> v3.Bucket: try: return super().get_bucket_by_id(bucket_id) except v3BucketIdNotFound as e: raise BucketIdNotFound(e.bucket_id) # one could contemplate putting "@limit_trace_arguments(only=('self', 'realm'))" here but logfury meta magic copies # the appropriate attributes from base classes def authorize_account(self, realm, application_key_id, application_key): return super().authorize_account( application_key_id=application_key_id, application_key=application_key, realm=realm, ) b2-sdk-python-2.8.0/b2sdk/v2/b2http.py000066400000000000000000000014441474454370000172460ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/b2http.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import _v3 as v3 from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound from .exception import BucketIdNotFound # Overridden to retain old-style BadRequest exception in case of a bad bucket id class B2Http(v3.B2Http): @classmethod def _translate_errors(cls, fcn, post_params=None): try: return super()._translate_errors(fcn, post_params) except v3BucketIdNotFound as e: raise BucketIdNotFound(e.bucket_id) b2-sdk-python-2.8.0/b2sdk/v2/bucket.py000066400000000000000000000142221474454370000173160ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/bucket.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import typing from b2sdk import _v3 as v3 from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound from b2sdk.v2._compat import _file_infos_rename from b2sdk._internal.http_constants import LIST_FILE_NAMES_MAX_LIMIT from .exception import BucketIdNotFound from .file_version import FileVersionFactory if typing.TYPE_CHECKING: from b2sdk._internal.utils import Sha1HexDigest from b2sdk._internal.filter import Filter from .file_version import FileVersion # Overridden to raise old style BucketIdNotFound exception class Bucket(v3.Bucket): FILE_VERSION_FACTORY_CLASS = staticmethod(FileVersionFactory) def get_fresh_state(self) -> Bucket: try: return super().get_fresh_state() except v3BucketIdNotFound as e: raise BucketIdNotFound(e.bucket_id) @_file_infos_rename def upload_bytes( self, data_bytes, file_name, content_type=None, file_info: dict | None = None, progress_listener=None, encryption: v3.EncryptionSetting | None = None, file_retention: v3.FileRetentionSetting | None = None, legal_hold: v3.LegalHold | None = None, large_file_sha1: Sha1HexDigest | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, *args, **kwargs, ): return super().upload_bytes( data_bytes, file_name, content_type, file_info, progress_listener, encryption, file_retention, legal_hold, large_file_sha1, custom_upload_timestamp, cache_control, *args, **kwargs, ) @_file_infos_rename def upload_local_file( self, local_file, file_name, content_type=None, file_info: dict | None = None, sha1_sum=None, min_part_size=None, progress_listener=None, encryption: v3.EncryptionSetting | None = None, file_retention: v3.FileRetentionSetting | None = None, legal_hold: v3.LegalHold | None = None, upload_mode: v3.UploadMode = v3.UploadMode.FULL, custom_upload_timestamp: int | None = None, cache_control: str | None = None, *args, **kwargs, ): return super().upload_local_file( local_file, file_name, content_type, file_info, sha1_sum, min_part_size, progress_listener, encryption, file_retention, legal_hold, upload_mode, custom_upload_timestamp, cache_control, *args, **kwargs, ) def ls( self, folder_to_list: str = '', latest_only: bool = True, recursive: bool = False, fetch_count: int | None = LIST_FILE_NAMES_MAX_LIMIT, with_wildcard: bool = False, filters: typing.Sequence[Filter] = (), folder_to_list_can_be_a_file: bool = False, **kwargs, ) -> typing.Iterable[tuple[FileVersion, str]]: """ Pretend that folders exist and yields the information about the files in a folder. B2 has a flat namespace for the files in a bucket, but there is a convention of using "/" as if there were folders. This method searches through the flat namespace to find the files and "folders" that live within a given folder. When the `recursive` flag is set, lists all of the files in the given folder, and all of its sub-folders. :param folder_to_list: the name of the folder to list; must not start with "/". Empty string means top-level folder :param latest_only: when ``False`` returns info about all versions of a file, when ``True``, just returns info about the most recent versions :param recursive: if ``True``, list folders recursively :param fetch_count: how many entries to list per API call or ``None`` to use the default. Acceptable values: 1 - 10000 :param with_wildcard: Accepts "*", "?", "[]" and "[!]" in folder_to_list, similarly to what shell does. As of 1.19.0 it can only be enabled when recursive is also enabled. Also, in this mode, folder_to_list is considered to be a filename or a pattern. :param filters: list of filters to apply to the files returned by the server. :param folder_to_list_can_be_a_file: if ``True``, folder_to_list can be a file, not just a folder This enabled default behavior of b2sdk.v3.Bucket.ls, in which for all paths that do not end with '/', first we try to check if file with this exact name exists, and only if it does not then we try to list files with this prefix. :rtype: generator[tuple[b2sdk.v2.FileVersion, str]] :returns: generator of (file_version, folder_name) tuples .. note:: In case of `recursive=True`, folder_name is not returned. """ if ( not folder_to_list_can_be_a_file and folder_to_list and not folder_to_list.endswith('/') and not with_wildcard ): folder_to_list += '/' yield from super().ls( path=folder_to_list, latest_only=latest_only, recursive=recursive, fetch_count=fetch_count, with_wildcard=with_wildcard, filters=filters, **kwargs, ) # Overridden to use old style Bucket class BucketFactory(v3.BucketFactory): BUCKET_CLASS = staticmethod(Bucket) b2-sdk-python-2.8.0/b2sdk/v2/exception.py000066400000000000000000000014431474454370000200400ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/exception.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._v3.exception import * # noqa v3BucketIdNotFound = BucketIdNotFound UnSyncableFilename = UnsupportedFilename # overridden to retain old style isinstance check and attributes class BucketIdNotFound(v3BucketIdNotFound, BadRequest): def __init__(self, bucket_id): super().__init__(bucket_id) self.message = f'Bucket with id={bucket_id} not found' self.code = 'bad_bucket_id' def __str__(self): return BadRequest.__str__(self) b2-sdk-python-2.8.0/b2sdk/v2/file_version.py000066400000000000000000000055261474454370000205340ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/file_version.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from typing import TYPE_CHECKING from b2sdk.v2 import EncryptionSetting from b2sdk.v2 import NO_RETENTION_FILE_SETTING, FileRetentionSetting, LegalHold from b2sdk.v2 import ReplicationStatus from b2sdk import _v3 as v3 if TYPE_CHECKING: from .api import B2Api UNVERIFIED_CHECKSUM_PREFIX = 'unverified:' class FileVersion(v3.FileVersion): """ A structure which represents a version of a file (in B2 cloud). :ivar str ~.id_: ``fileId`` :ivar str ~.file_name: full file name (with path) :ivar ~.size: size in bytes, can be ``None`` (unknown) :ivar str ~.content_type: RFC 822 content type, for example ``"application/octet-stream"`` :ivar ~.upload_timestamp: in milliseconds since :abbr:`epoch (1970-01-01 00:00:00)`. Can be ``None`` (unknown). :ivar str ~.action: ``"upload"``, ``"hide"`` or ``"delete"`` """ __slots__ = ['cache_control'] def __init__( self, api: B2Api, id_: str, file_name: str, size: int | None | str, content_type: str | None, content_sha1: str | None, file_info: dict[str, str], upload_timestamp: int, account_id: str, bucket_id: str, action: str, content_md5: str | None, server_side_encryption: EncryptionSetting, file_retention: FileRetentionSetting = NO_RETENTION_FILE_SETTING, legal_hold: LegalHold = LegalHold.UNSET, replication_status: ReplicationStatus | None = None, cache_control: str | None = None, ): self.cache_control = cache_control if self.cache_control is None: self.cache_control = (file_info or {}).get('b2-cache-control') super().__init__( api=api, id_=id_, file_name=file_name, size=size, content_type=content_type, content_sha1=content_sha1, file_info=file_info, upload_timestamp=upload_timestamp, account_id=account_id, bucket_id=bucket_id, action=action, content_md5=content_md5, server_side_encryption=server_side_encryption, file_retention=file_retention, legal_hold=legal_hold, replication_status=replication_status, ) def as_dict(self): result = super().as_dict() if self.cache_control is not None: result['cacheControl'] = self.cache_control return result class FileVersionFactory(v3.FileVersionFactory): FILE_VERSION_CLASS = FileVersion b2-sdk-python-2.8.0/b2sdk/v2/large_file.py000066400000000000000000000024731474454370000201370ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/large_file.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import _v3 as v3 class UnfinishedLargeFile(v3.UnfinishedLargeFile): """ A structure which represents a version of a file (in B2 cloud). :ivar str ~.file_id: ``fileId`` :ivar str ~.file_name: full file name (with path) :ivar str ~.account_id: account ID :ivar str ~.bucket_id: bucket ID :ivar str ~.content_type: :rfc:`822` content type, for example ``"application/octet-stream"`` :ivar dict ~.file_info: file info dict """ # In v3, cache_control is a property. # We set this to None so that it can be assigned to in __init__. cache_control = None def __init__(self, file_dict): """ Initialize from one file returned by ``b2_start_large_file`` or ``b2_list_unfinished_large_files``. """ super().__init__(file_dict) self.cache_control = (file_dict['fileInfo'] or {}).get('b2-cache-control') class LargeFileServices(v3.LargeFileServices): UNFINISHED_LARGE_FILE_CLASS = staticmethod(UnfinishedLargeFile) b2-sdk-python-2.8.0/b2sdk/v2/raw_api.py000066400000000000000000000053011474454370000174610ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/raw_api.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import _v3 as v3 from b2sdk.v2._compat import _file_infos_rename class _OldRawAPI: """RawAPI compatibility layer""" @classmethod @_file_infos_rename def get_upload_file_headers( cls, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_info: dict, server_side_encryption: v3.EncryptionSetting | None, file_retention: v3.FileRetentionSetting | None, legal_hold: v3.LegalHold | None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, *args, **kwargs, ) -> dict: if cache_control is not None: file_info['b2-cache-control'] = cache_control return super().get_upload_file_headers( upload_auth_token, file_name, content_length, content_type, content_sha1, file_info, server_side_encryption, file_retention, legal_hold, custom_upload_timestamp, *args, **kwargs, ) def unprintable_to_hex(self, s): return v3.unprintable_to_hex(s) @_file_infos_rename def upload_file( self, upload_url, upload_auth_token, file_name, content_length, content_type, content_sha1, file_info: dict, data_stream, server_side_encryption: v3.EncryptionSetting | None = None, file_retention: v3.FileRetentionSetting | None = None, legal_hold: v3.LegalHold | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, *args, **kwargs, ): if cache_control is not None: file_info['b2-cache-control'] = cache_control return super().upload_file( upload_url, upload_auth_token, file_name, content_length, content_type, content_sha1, file_info, data_stream, server_side_encryption, file_retention, legal_hold, custom_upload_timestamp, *args, **kwargs, ) class AbstractRawApi(_OldRawAPI, v3.AbstractRawApi): pass class B2RawHTTPApi(_OldRawAPI, v3.B2RawHTTPApi): pass b2-sdk-python-2.8.0/b2sdk/v2/raw_simulator.py000066400000000000000000000071731474454370000207400ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/raw_simulator.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import _v3 as v3 from b2sdk.v2._compat import _file_infos_rename class BucketSimulator(v3.BucketSimulator): @_file_infos_rename def upload_file( self, upload_id: str, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_info: dict, data_stream, server_side_encryption: v3.EncryptionSetting | None = None, file_retention: v3.FileRetentionSetting | None = None, legal_hold: v3.LegalHold | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, *args, **kwargs, ): if cache_control is not None: file_info['b2-cache-control'] = cache_control return super().upload_file( upload_id, upload_auth_token, file_name, content_length, content_type, content_sha1, file_info, data_stream, server_side_encryption, file_retention, legal_hold, custom_upload_timestamp, *args, **kwargs, ) class RawSimulator(v3.RawSimulator): @classmethod @_file_infos_rename def get_upload_file_headers( cls, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_info: dict, server_side_encryption: v3.EncryptionSetting | None, file_retention: v3.FileRetentionSetting | None, legal_hold: v3.LegalHold | None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, *args, **kwargs, ) -> dict: if cache_control is not None: file_info['b2-cache-control'] = cache_control return super().get_upload_file_headers( upload_auth_token, file_name, content_length, content_type, content_sha1, file_info, server_side_encryption, file_retention, legal_hold, custom_upload_timestamp, *args, **kwargs, ) @_file_infos_rename def upload_file( self, upload_url: str, upload_auth_token: str, file_name: str, content_length: int, content_type: str, content_sha1: str, file_info: dict, data_stream, server_side_encryption: v3.EncryptionSetting | None = None, file_retention: v3.FileRetentionSetting | None = None, legal_hold: v3.LegalHold | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, *args, **kwargs, ): if cache_control is not None: file_info['b2-cache-control'] = cache_control return super().upload_file( upload_url, upload_auth_token, file_name, content_length, content_type, content_sha1, file_info, data_stream, server_side_encryption, file_retention, legal_hold, custom_upload_timestamp, *args, **kwargs, ) b2-sdk-python-2.8.0/b2sdk/v2/session.py000066400000000000000000000042141474454370000175240ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/session.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import _v3 as v3 from .b2http import B2Http from ._compat import _file_infos_rename from .._internal import api_config as _api_config from .._internal import cache as _cache from .._internal.account_info import abstract as _abstract # Override to use legacy B2Http class B2Session(v3.B2Session): B2HTTP_CLASS = staticmethod(B2Http) def __init__( self, account_info: _abstract.AbstractAccountInfo | None = None, cache: _cache.AbstractCache | None = None, api_config: _api_config.B2HttpApiConfig = _api_config.DEFAULT_HTTP_API_CONFIG, ): if account_info is not None and cache is None: # preserve legacy behavior https://github.com/Backblaze/b2-sdk-python/issues/497#issuecomment-2147461352 cache = _cache.DummyCache() super().__init__(account_info, cache, api_config) @_file_infos_rename def upload_file( self, bucket_id, file_name, content_length, content_type, content_sha1, file_info, data_stream, server_side_encryption: v3.EncryptionSetting | None = None, file_retention: v3.FileRetentionSetting | None = None, legal_hold: v3.LegalHold | None = None, custom_upload_timestamp: int | None = None, cache_control: str | None = None, *args, **kwargs, ): if cache_control is not None: file_info['b2-cache-control'] = cache_control return super().upload_file( bucket_id, file_name, content_length, content_type, content_sha1, file_info, data_stream, server_side_encryption, file_retention, legal_hold, custom_upload_timestamp, *args, **kwargs, ) b2-sdk-python-2.8.0/b2sdk/v2/sync.py000066400000000000000000000005611474454370000170160ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/sync.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._v3 import B2Path B2SyncPath = B2Path b2-sdk-python-2.8.0/b2sdk/v2/transfer.py000066400000000000000000000011101474454370000176550ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/transfer.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import _v3 as v3 from b2sdk._internal.utils.thread_pool import LazyThreadPool # noqa: F401 class ThreadPoolMixin(v3.ThreadPoolMixin): pass class DownloadManager(v3.DownloadManager): pass class UploadManager(v3.UploadManager): pass b2-sdk-python-2.8.0/b2sdk/v2/utils.py000066400000000000000000000020111474454370000171720ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/utils.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import shutil import tempfile import warnings class TempDir: """ Context manager that creates and destroys a temporary directory. """ def __enter__(self): """ Return the unicode path to the temp dir. """ warnings.warn( 'TempDir is deprecated. Use tempfile.TemporaryDirectory or pytest tmp_path fixture instead.', DeprecationWarning, stacklevel=2, ) dirpath_bytes = tempfile.mkdtemp() self.dirpath = str(dirpath_bytes.replace('\\', '\\\\')) return self.dirpath def __exit__(self, exc_type, exc_val, exc_tb): shutil.rmtree(self.dirpath) return None # do not hide exception b2-sdk-python-2.8.0/b2sdk/v2/version_utils.py000066400000000000000000000026601474454370000207510ustar00rootroot00000000000000###################################################################### # # File: b2sdk/v2/version_utils.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk import _v3 as v3 class _OldAbstractDeprecatorMixin: def __call__(self, *args, **kwargs): if self.cutoff_version: assert ( self.current_version < self.cutoff_version ), f'{self.__class__.__name__} decorator is still used in version {self.current_version} when old {self.WHAT} name {self.source!r} was scheduled to be dropped in {self.cutoff_version}. It is time to remove the mapping.' ret = super().__call__(*args, **kwargs) assert ( self.changed_version <= self.current_version ), f'{self.__class__.__name__} decorator indicates that the replacement of {self.WHAT} {self.source!r} should take place in the future version {self.changed_version}, while the current version is {self.cutoff_version}. It looks like should be _discouraged_ at this point and not _deprecated_ yet. Consider using {self.ALTERNATIVE_DECORATOR!r} decorator instead.' return ret class rename_argument(_OldAbstractDeprecatorMixin, v3.rename_argument): pass class rename_function(_OldAbstractDeprecatorMixin, v3.rename_function): pass b2-sdk-python-2.8.0/b2sdk/version.py000066400000000000000000000012241474454370000171750ustar00rootroot00000000000000###################################################################### # # File: b2sdk/version.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from importlib.metadata import version as _version from sys import version_info as _version_info __all__ = [ 'VERSION', 'PYTHON_VERSION', 'USER_AGENT', ] VERSION = _version('b2sdk') PYTHON_VERSION = '.'.join(map(str, _version_info[:3])) # something like: 3.9.1 USER_AGENT = f'backblaze-b2/{VERSION} python/{PYTHON_VERSION}' b2-sdk-python-2.8.0/changelog.d/000077500000000000000000000000001474454370000163035ustar00rootroot00000000000000b2-sdk-python-2.8.0/changelog.d/.gitkeep000066400000000000000000000000001474454370000177220ustar00rootroot00000000000000b2-sdk-python-2.8.0/contrib/000077500000000000000000000000001474454370000155725ustar00rootroot00000000000000b2-sdk-python-2.8.0/contrib/color-b2-logs.sh000077500000000000000000000007461474454370000205210ustar00rootroot00000000000000#!/bin/bash -eu awk -F '\t' '{print $1 " " $4 " " $5 " " $6}' | colorex --green=DEBUG \ --bgreen=INFO \ --bred=ERROR \ --byellow=WARNING \ --bmagenta='calling [\w\.]+' \ --bblue='INFO // =+ [0-9\.]+ =+ \\' \ --bblue='INFO // =+ [0-9\.]+ =+ \\' \ --bblue='starting command .* with arguments:' \ --bblue='starting command .* \(arguments hidden\)' \ --red=Traceback \ --green='\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d' \ --cyan='b2\.sync' b2-sdk-python-2.8.0/contrib/debug_logs.ini000066400000000000000000000013071474454370000204060ustar00rootroot00000000000000############################################################ [loggers] keys=root,b2 [logger_root] level=DEBUG handlers=fileHandler [logger_b2] level=DEBUG handlers=fileHandler qualname=b2 propagate=0 ############################################################ [handlers] keys=fileHandler [handler_fileHandler] class=logging.handlers.TimedRotatingFileHandler level=DEBUG formatter=simpleFormatter args=('b2sdk.log', 'midnight') ############################################################ [formatters] keys=simpleFormatter [formatter_simpleFormatter] format=%(asctime)s %(process)d %(thread)d %(name)s %(levelname)s %(message)s datefmt= ############################################################ b2-sdk-python-2.8.0/doc/000077500000000000000000000000001474454370000146775ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/bash_completion.md000066400000000000000000000021561474454370000203730ustar00rootroot00000000000000In order to use bash completion, you can have a `~/.bash_completion` like this: ```sh if [ -d "$HOME/.bash_completion.d" ]; then for file in "$HOME/.bash_completion.d/"* do source "$file" >/dev/null 2>&1 done fi ``` and then copy our `contrib/bash_completion/b2` to your `~/.bash_completion.d/`. The important trick is that `b2` tool must be in PATH before bash_completions are loaded for the last time (unless you delete the first line of our completion script). If you keep the `b2` tool in `~/bin`, you can make sure the loading order is proper by making sure `~/bin` is added to the PATH before loading bash_completion. To do that, add the following snippet to your `~/.bashrc`: ```sh if [ -d ~/bin ]; then PATH="$HOME/bin:$PATH" fi # enable programmable completion features (you don't need to enable # this, if it's already enabled in /etc/bash.bashrc and /etc/profile # sources /etc/bash.bashrc). if ! shopt -oq posix; then if [ -f /usr/share/bash-completion/bash_completion ]; then . /usr/share/bash-completion/bash_completion elif [ -f /etc/bash_completion ]; then . /etc/bash_completion fi fi ``` b2-sdk-python-2.8.0/doc/markup-test.rst000066400000000000000000000107651474454370000177160ustar00rootroot00000000000000Test ~~~~ .. DANGER:: Beware killer rabbits! .. note:: This is a note admonition. This is the second line of the first paragraph. - The note contains all indented body elements following. - It includes this bullet list. .. admonition:: And, by the way... You can make up your own admonition too. .. title:: This is title block of text .. admonition:: This is admonition block of text .. warning:: This is warning block of text .. tip:: This is tip block of text .. note:: This is note block of text .. important:: This is important block of text .. hint:: This is hint block of text .. error:: This is error block of text .. danger:: This is danger block of text .. caution:: This is caution block of text .. attention:: This is attention block of text .. _semver: Types of interfaces and version pinning ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ b2sdk is divided into three parts. Please pay attention to which group you use, as the stability of your application depends on correct pinning of versions. Public ~~~~~~ Public interface consists of *public* members of the following modules: * b2sdk.api.B2Api * b2sdk.bucket.Bucket * b2sdk.exception * b2sdk.sync * b2sdk.sync.exception and some of their dependencies: * b2sdk._internal.account_info.InMemoryAccountInfo * b2sdk._internal.account_info.SqliteAccountInfo * b2sdk.transferer * b2sdk._internal.utils Those will not change in a backwards-incompatible way between non-major versions. In other words, if you pin your dependencies to `>=x.0.0;=4.5.6;<5.0.0 .. note:: b2sdk.*._something and b2sdk.*.*._something, having a name which begins with an underscore, are NOT considered public interface. Protected ~~~~~~~~~ Things which sometimes might be necessary to use that are NOT considered public interface (and may change in a non-major version): * B2Session * B2RawHTTPApi * B2Http .. note:: it is ok for you to use those (better that, than copying our sources), however if you do, please pin your dependencies to middle version. .. example:: If the current version of b2sdk is 4.5.6 and you use the public and protected interfaces, put this in your requirements.txt: >=4.5.6;<4.6.0 Private ~~~~~~~ If you need to use some of our private interfaces, pin your dependencies strictly. .. example:: If the current version of b2sdk is 4.5.6 and you use the private interface, put this in your requirements.txt: ==4.5.6 Authorization ~~~~~~~~~~~~~ Before you can use b2sdk, you need to prove who you are to the server. For that you will need to pass `account id` and `api token` to one of the authorization classes. In case you are storing that information in a database or something, you can implement your own class by inheriting from AbstractAuthorization. Otherwise, use one of the classes included in b2sdk package: InMemoryAccountInfo: This is probably what your application should be using and also what we use in our tests. SqliteAccountInfo: this is what B2 CLI uses to authorize the user. Stores information in a local file. B2Api ~~~~ The "main" object that abstracts the communication with B2 cloud is B2Api. It lets you manage buckets and download files by id. example Bucket ~~~~~~ Bucket class abstracts the B2 bucket, which is essentially a namespace for objects. The best way to transfer your files into a bucket and back, is to use *sync*. If for some reason you cannot use sync, it is also possible to upload and download files directly into/from the bucket, using Bucket.upload_file and Bucket.download_by_name. The Bucket object also contains a few methods to list the contents of the bucket and the metadata associated with the objects contained in it. ======== Tutorial ======== Account authorization ===================== TODO Bucket actions ============== Create a bucket --------------- TODO Remove a bucket --------------- TODO List buckets ------------- TODO Update bucket info ------------------ TODO File actions ============ Upload file ----------- TODO Download file ------------- TODO List files ---------- TODO Get file meta information ------------------------- TODO Delete file ----------- TODO Cancel file operations ---------------------- TODO Synchronization =============== TODO Account information =================== TODO b2-sdk-python-2.8.0/doc/render_sqlite_account_info_schema.sh000077500000000000000000000001241474454370000241420ustar00rootroot00000000000000python ../sqlite_account_info_schema.py > source/dot/sqlite_account_info_schema.dot b2-sdk-python-2.8.0/doc/source/000077500000000000000000000000001474454370000161775ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/advanced.rst000066400000000000000000000527361474454370000205130ustar00rootroot00000000000000.. _AdvancedUsagePatterns: ######################################### Advanced usage patterns ######################################### B2 server API allows for creation of an object from existing objects. This allows to avoid transferring data from the source machine if the desired outcome can be (at least partially) constructed from what is already on the server. The way **b2sdk** exposes this functionality is through a few functions that allow the user to express the desired outcome and then the library takes care of planning and executing the work. Please refer to the table below to compare the support of object creation methods for various usage patterns. ***************** Available methods ***************** .. |br| raw:: html
.. _advanced_methods_support_table: +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | Method / supported options | Source | Range |br| overlap | Streaming |br| interface | :ref:`Continuation ` | +============================================+========+=====================+==========================+====================================+ | :meth:`b2sdk.v2.Bucket.upload` | local | no | no | automatic | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | :meth:`b2sdk.v2.Bucket.copy` | remote | no | no | automatic | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | :meth:`b2sdk.v2.Bucket.concatenate` | any | no | no | automatic | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | :meth:`b2sdk.v2.Bucket.concatenate_stream` | any | no | yes | manual | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | :meth:`b2sdk.v2.Bucket.create_file` | any | yes | no | automatic | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ | :meth:`b2sdk.v2.Bucket.create_file_stream` | any | yes | yes | manual | +--------------------------------------------+--------+---------------------+--------------------------+------------------------------------+ Range overlap ============= Some methods support overlapping ranges between local and remote files. **b2sdk** tries to use the remote ranges as much as possible, but due to limitations of ``b2_copy_part`` (specifically the minimum size of a part) that may not be always possible. A possible solution for such case is to download a (small) range and then upload it along with another one, to meet the ``b2_copy_part`` requirements. This can be improved if the same data is already available locally - in such case **b2sdk** will use the local range rather than downloading it. Streaming interface =================== Some object creation methods start writing data before reading the whole input (iterator). This can be used to write objects that do not have fully known contents without writing them first locally, so that they could be copied. Such usage pattern can be relevant to small devices which stream data to B2 from an external NAS, where caching large files such as media files or virtual machine images is not an option. Please see :ref:`advanced method support table ` to see where streaming interface is supported. Continuation ============ Please see :ref:`here ` ***************** Concatenate files ***************** :meth:`b2sdk.v2.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). It can be used to glue remote files together, back-to-back, into a new file. :meth:`b2sdk.v2.Bucket.concatenate_stream` does not create and validate a plan before starting the transfer, so it can be used to process a large input iterator, at a cost of limited automated continuation. Concatenate files of known size ================================= .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> input_sources = [ ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, length=100), ... UploadSourceLocalFile('my_local_path/to_file.txt'), ... CopySource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.concatenate(input_sources, remote_name, file_info) If one of remote source has length smaller than :term:`absoluteMinimumPartSize` then it cannot be copied into large file part. Such remote source would be downloaded and concatenated locally with local source or with other downloaded remote source. Please note that this method only allows checksum verification for local upload sources. Checksum verification for remote sources is available only when local copy is available. In such case :meth:`b2sdk.v2.Bucket.create_file` can be used with overalapping ranges in input. For more information about ``concatenate`` please see :meth:`b2sdk.v2.Bucket.concatenate` and :class:`b2sdk.v2.CopySource`. Concatenate files of known size (streamed version) ================================================== :meth:`b2sdk.v2.Bucket.concatenate` accepts an iterable of upload sources (either local or remote). The operation would not be planned ahead so it supports very large output objects, but continuation is only possible for local only sources and provided unfinished large file id. See more about continuation in :meth:`b2sdk.v2.Bucket.create_file` paragraph about continuation. .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> input_sources = [ ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=100, length=100), ... UploadSourceLocalFile('my_local_path/to_file.txt'), ... CopySource('4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', length=2123456789), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.concatenate_stream(input_sources, remote_name, file_info) Concatenate files of unknown size ================================= While it is supported by B2 server, this pattern is currently not supported by **b2sdk**. ********************* Synthethize an object ********************* Using methods described below an object can be created from both local and remote sources while avoiding downloading small ranges when such range is already present on a local drive. Update a file efficiently ==================================== :meth:`b2sdk.v2.Bucket.create_file` accepts an iterable which *can contain overlapping destination ranges*. .. note:: Following examples *create* new file - data in bucket is immutable, but **b2sdk** can create a new file version with the same name and updated content Append to the end of a file --------------------------- The assumption here is that the file has been appended to since it was last uploaded to. This assumption is verified by **b2sdk** when possible by recalculating checksums of the overlapping remote and local ranges. If copied remote part sha does not match with locally available source, file creation process would be interrupted and an exception would be raised. .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> input_sources = [ ... WriteIntent( ... data=CopySource( ... '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', ... offset=0, ... length=5000000, ... ), ... destination_offset=0, ... ), ... WriteIntent( ... data=UploadSourceLocalFile('my_local_path/to_file.txt'), # of length 60000000 ... destination_offset=0, ... ), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.create_file(input_sources, remote_name, file_info) `LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v2.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range (when concatenating, local source would have destination offset at the end of remote source) For more information see :meth:`b2sdk.v2.Bucket.create_file`. Change the middle of the remote file ------------------------------------ .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> input_sources = [ ... WriteIntent( ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=4000000), ... destination_offset=0, ... ), ... WriteIntent( ... UploadSourceLocalFile('my_local_path/to_file.txt'), # length=1024, here not passed and just checked from local source using seek ... destination_offset=4000000, ... ), ... WriteIntent( ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=4001024, length=123456789), ... destination_offset=4001024, ... ), ... ] >>> file_info = {'how': 'good-file'} >>> bucket.create_file(input_sources, remote_name, file_info) `LocalUploadSource` has the size determined automatically in this case. This is more efficient than :meth:`b2sdk.v2.Bucket.concatenate`, as it can use the overlapping ranges when a remote part is smaller than :term:`absoluteMinimumPartSize` to prevent downloading a range. For more information see :meth:`b2sdk.v2.Bucket.create_file`. Synthesize a file from local and remote parts ============================================= This is useful for expert usage patterns such as: - *synthetic backup* - *reverse synthetic backup* - mostly-server-side cutting and gluing uncompressed media files such as `wav` and `avi` with rewriting of file headers - various deduplicated backup scenarios Please note that :meth:`b2sdk.v2.Bucket.create_file_stream` accepts **an ordered iterable** which *can contain overlapping ranges*, so the operation does not need to be planned ahead, but can be streamed, which supports very large output objects. Scenarios such as below are then possible: .. code-block:: A C D G | | | | | cloud-AC | | cloud-DG | | | | | v v v v ############ ############# ^ ^ | | +---- desired file A-G --------+ | | | | | ######################### | | ^ ^ | | | | | | | local file-BF | | | | | | A B C D E F G .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> def generate_input(): ... yield WriteIntent( ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=lengthC), ... destination_offset=0, ... ) ... yield WriteIntent( ... UploadSourceLocalFile('my_local_path/to_file.txt'), # length = offsetF - offsetB ... destination_offset=offsetB, ... ) ... yield WriteIntent( ... CopySource('4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', offset=0, length=offsetG-offsetD), ... destination_offset=offsetD, ... ) ... >>> file_info = {'how': 'good-file'} >>> bucket.create_file(generate_input(), remote_name, file_info) In such case, if the sizes allow for it (there would be no parts smaller than :term:`absoluteMinimumPartSize`), the only uploaded part will be `C-D`. Otherwise, more data will be uploaded, but the data transfer will be reduced in most cases. :meth:`b2sdk.v2.Bucket.create_file` does not guarantee that outbound transfer usage would be optimal, it uses a simple greedy algorithm with as small look-aheads as possible. For more information see :meth:`b2sdk.v2.Bucket.create_file`. Encryption ---------- Even if files `A-C` and `D-G` are encrypted using `SSE-C` with different keys, they can still be used in a single :meth:`b2sdk.v2.Bucket.create_file` call, because :class:`b2sdk.v2.CopySource` accepts an optional :class:`b2sdk.v2.EncryptionSetting`. Prioritize remote or local sources ---------------------------------- :meth:`b2sdk.v2.Bucket.create_file` and :meth:`b2sdk.v2.Bucket.create_file_stream` support source/origin prioritization, so that planner would know which sources should be used for overlapping ranges. Supported values are: `local`, `remote` and `local_verification`. .. code-block:: A D G | | | | cloud-AD | | | | | v v | ################ | ^ | | | +---- desired file A-G --------+ | | | | | ####### ################# | ^ ^ ^ | | | | | | | | local file BC and DE | | | | | | A B C D E A=0, B=50M, C=80M, D=100M, E=200 .. code-block:: python >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local') # planner parts: cloud[A, B], local[B, C], remote[C, D], local[D, E] Here the planner has only used a remote source where remote range was not available, minimizing downloads. .. code-block:: python >>> planner.create_file(input_sources, remote_name, file_info, prioritize='remote') # planner parts: cloud[A, D], local[D, E] Here the planner has only used a local source where remote range was not available, minimizing uploads. .. code-block:: python >>> bucket.create_file(input_sources, remote_name, file_info) # or >>> bucket.create_file(input_sources, remote_name, file_info, prioritize='local_verification') # planner parts: cloud[A, B], cloud[B, C], cloud[C, D], local[D, E] In `local_verification` mode the remote range was artificially split into three parts to allow for checksum verification against matching local ranges. .. note:: `prioritize` is just a planner setting - remote parts are always verified if matching local parts exists. .. TODO:: prioritization should accept enum, not string .. _continuation: ************ Continuation ************ Continuation of upload ====================== In order to continue a simple upload session, **b2sdk** checks for any available sessions with of the same ``file name``, ``file_info`` and ``media type``, verifying the size of an object as much as possible. To support automatic continuation, some advanced methods create a plan before starting copy/upload operations, saving the hash of that plan in ``file_info`` for increased reliability. If that is not available, ``large_file_id`` can be extracted via callback during the operation start. It can then be passed into the subsequent call to continue the same task, though the responsibility for passing the exact same input is then on the user of the function. Please see :ref:`advanced method support table ` to see where automatic continuation is supported. ``large_file_id`` can also be passed if automatic continuation is available in order to avoid issues where multiple matching upload sessions are matching the transfer. Continuation of create/concatenate =================================== :meth:`b2sdk.v2.Bucket.create_file` supports automatic continuation or manual continuation. :meth:`b2sdk.v2.Bucket.create_file_stream` supports only manual continuation for local-only inputs. The situation looks the same for :meth:`b2sdk.v2.Bucket.concatenate` and :meth:`b2sdk.v2.Bucket.concatenate_stream` (streamed version supports only manual continuation of local sources). Also :meth:`b2sdk.v2.Bucket.upload` and :meth:`b2sdk.v2.Bucket.copy` support both automatic and manual continuation. Manual continuation ------------------- .. code-block:: python >>> def large_file_callback(large_file): ... # storage is not part of the interface - here only for demonstration purposes ... storage.store({'name': remote_name, 'large_file_id': large_file.id}) >>> bucket.create_file(input_sources, remote_name, file_info, large_file_callback=large_file_callback) # ... >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] >>> bucket.create_file(input_sources, remote_name, file_info, large_file_id=large_file_id) Manual continuation (streamed version) -------------------------------------- .. code-block:: python >>> def large_file_callback(large_file): ... # storage is not part of the interface - here only for demonstration purposes ... storage.store({'name': remote_name, 'large_file_id': large_file.id}) >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_callback=large_file_callback) # ... >>> large_file_id = storage.query({'name': remote_name})[0]['large_file_id'] >>> bucket.create_file_stream(input_sources, remote_name, file_info, large_file_id=large_file_id) Streams that contains remote sources cannot be continued with :meth:`b2sdk.v2.Bucket.create_file` - internally :meth:`b2sdk.v2.Bucket.create_file` stores plan information in file info for such inputs, and verifies it before any copy/upload and :meth:`b2sdk.v2.Bucket.create_file_stream` cannot store this information. Local source only inputs can be safely continued with :meth:`b2sdk.v2.Bucket.create_file` in auto continue mode or manual continue mode (because plan information is not stored in file info in such case). Auto continuation ----------------- .. code-block:: python >>> bucket.create_file(input_sources, remote_name, file_info) For local source only input, :meth:`b2sdk.v2.Bucket.create_file` would try to find matching unfinished large file. It will verify uploaded parts checksums with local sources - the most completed, having all uploaded parts matched candidate would be automatically selected as file to continue. If there is no matching candidate (even if there are unfinished files for the same file name) new large file would be started. In other cases plan information would be generated and :meth:`b2sdk.v2.Bucket.create_file` would try to find unfinished large file with matching plan info in its file info. If there is one or more such unfinished large files, :meth:`b2sdk.v2.Bucket.create_file` would verify checksums for all locally available parts and choose any matching candidate. If all candidates fails on uploaded parts checksums verification, process is interrupted and error raises. In such case corrupted unfinished large files should be cancelled manullay and :meth:`b2sdk.v2.Bucket.create_file` should be retried, or auto continuation should be turned off with `auto_continue=False` No continuation --------------- .. code-block:: python >>> bucket.create_file(input_sources, remote_name, file_info, auto_continue=False) Note, that this only forces start of a new large file - it is still possible to continue the process with either auto or manual modes. **************************** SHA-1 hashes for large files **************************** Depending on the number and size of sources and the size of the result file, the SDK may decide to use the large file API to create a file on the server. In such cases the file's SHA-1 won't be stored on the server in the ``X-Bz-Content-Sha1`` header, but it may optionally be stored with the file in the ``large_file_sha1`` entry in the ``file_info``, as per [B2 integration checklist](https://www.backblaze.com/b2/docs/integration_checklist.html). In basic scenarios, large files uploaded to the server will have a ``large_file_sha1`` element added automatically to their ``file_info``. However, when concatenating multiple sources, it may be impossible for the SDK to figure out the SHA-1 automatically. In such cases, the SHA-1 can be provided using the ``large_file_sha1`` parameter to :meth:`b2sdk.v2.Bucket.create_file`, :meth:`b2sdk.v2.Bucket.concatenate` and their stream equivalents. If the parameter is skipped or ``None``, the result file may not have the ``large_file_sha1`` value set. Note that the provided SHA-1 value is not verified. b2-sdk-python-2.8.0/doc/source/api/000077500000000000000000000000001474454370000167505ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/account_info.rst000066400000000000000000000104731474454370000221560ustar00rootroot00000000000000.. _AccountInfo: ######################## AccountInfo ######################## *AccountInfo* stores basic information about the account, such as *Application Key ID* and *Application Key*, in order to let :py:class:`b2sdk.v2.B2Api` perform authenticated requests. There are two usable implementations provided by **b2sdk**: * :py:class:`b2sdk.v2.InMemoryAccountInfo` - a basic implementation with no persistence * :py:class:`b2sdk.v2.SqliteAccountInfo` - for console and GUI applications They both provide the full :ref:`AccountInfo interface `. .. note:: Backup applications and many server-side applications should :ref:`implement their own ` *AccountInfo*, backed by the metadata/configuration database of the application. *************************** AccountInfo implementations *************************** InMemoryAccountInfo =================== *AccountInfo* with no persistence. .. autoclass:: b2sdk.v2.InMemoryAccountInfo() :no-members: Implements all methods of :ref:`AccountInfo interface `. .. hint:: Usage of this class is appropriate for secure Web applications which do not wish to persist any user data. Using this class for applications such as CLI, GUI or backup is discouraged, as ``InMemoryAccountInfo`` does not write down the authorization token persistently. That would be slow, as it would force the application to retrieve a new one on every command/click/backup start. Furthermore - an important property of *AccountInfo* is caching the ``bucket_name:bucket_id`` mapping; in case of ``InMemoryAccountInfo`` the cache will be flushed between executions of the program. .. method:: __init__() The constructor takes no parameters. SqliteAccountInfo ================= .. autoclass:: b2sdk.v2.SqliteAccountInfo() :inherited-members: :no-members: :special-members: __init__ Implements all methods of :ref:`AccountInfo interface `. Uses a `SQLite database `_ for persistence and access synchronization between multiple processes. Not suitable for usage over NFS. Underlying database has the following schema: .. graphviz:: /dot/sqlite_account_info_schema.dot .. hint:: Usage of this class is appropriate for interactive applications installed on a user's machine (i.e.: CLI and GUI applications). Usage of this class **might** be appropriate for non-interactive applications installed on the user's machine, such as backup applications. An alternative approach that should be considered is to store the *AccountInfo* data alongside the configuration of the rest of the application. .. _my_account_info: ********************* Implementing your own ********************* When building a server-side application or a web service, you might want to implement your own *AccountInfo* class backed by a database. In such case, you should inherit from :py:class:`b2sdk.v2.UrlPoolAccountInfo`, which has groundwork for url pool functionality). If you cannot use it, inherit directly from :py:class:`b2sdk.v2.AbstractAccountInfo`. .. code-block:: python >>> from b2sdk.v2 import UrlPoolAccountInfo >>> class MyAccountInfo(UrlPoolAccountInfo): ... :py:class:`b2sdk.v2.AbstractAccountInfo` describes the interface, while :py:class:`b2sdk.v2.UrlPoolAccountInfo` and :py:class:`b2sdk.v2.UploadUrlPool` implement a part of the interface for in-memory upload token management. .. _account_info_interface: AccountInfo interface ===================== .. autoclass:: b2sdk.v2.AbstractAccountInfo() :inherited-members: :private-members: :exclude-members: _abc_cache, _abc_negative_cache, _abc_negative_cache_version, _abc_registry AccountInfo helper classes ========================== .. autoclass:: b2sdk.v2.UrlPoolAccountInfo() :inherited-members: :no-members: :members: BUCKET_UPLOAD_POOL_CLASS, LARGE_FILE_UPLOAD_POOL_CLASS .. caution:: This class is not part of the public interface. To find out how to safely use it, read :ref:`this `. .. autoclass:: b2sdk._internal.account_info.upload_url_pool.UploadUrlPool() :inherited-members: :private-members: .. caution:: This class is not part of the public interface. To find out how to safely use it, read :ref:`this `. b2-sdk-python-2.8.0/doc/source/api/api.rst000066400000000000000000000003271474454370000202550ustar00rootroot00000000000000B2 Api client ============= .. autoclass:: b2sdk.v2.B2Api() :inherited-members: :special-members: __init__ .. autoclass:: b2sdk.v2.B2HttpApiConfig() :inherited-members: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/application_key.rst000066400000000000000000000003551474454370000226600ustar00rootroot00000000000000B2 Application key ================== .. autoclass:: b2sdk.v2.ApplicationKey() :inherited-members: :special-members: __init__ .. autoclass:: b2sdk.v2.FullApplicationKey() :inherited-members: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/bucket.rst000066400000000000000000000001551474454370000207600ustar00rootroot00000000000000B2 Bucket ========= .. autoclass:: b2sdk.v2.Bucket() :inherited-members: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/cache.rst000066400000000000000000000007721474454370000205530ustar00rootroot00000000000000Cache ===== **b2sdk** caches the mapping between bucket name and bucket id, so that the user of the library does not need to maintain the mapping to call the api. .. autoclass:: b2sdk.v2.AbstractCache :inherited-members: .. autoclass:: b2sdk.v2.AuthInfoCache() :inherited-members: :special-members: __init__ .. autoclass:: b2sdk.v2.DummyCache() :inherited-members: :special-members: __init__ .. autoclass:: b2sdk.v2.InMemoryCache() :inherited-members: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/data_classes.rst000066400000000000000000000007121474454370000221300ustar00rootroot00000000000000Data classes ============ .. autoclass:: b2sdk.v2.FileVersion :inherited-members: :special-members: __dict__ .. autoclass:: b2sdk.v2.DownloadVersion :inherited-members: .. autoclass:: b2sdk.v2.FileIdAndName :inherited-members: :special-members: __dict__ .. autoclass:: b2sdk.v2.UnfinishedLargeFile() :no-members: .. autoclass:: b2sdk.v2.Part :no-members: .. autoclass:: b2sdk.v2.Range :no-members: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/downloaded_file.rst000066400000000000000000000001621474454370000226200ustar00rootroot00000000000000Downloaded File =============== .. autoclass:: b2sdk.v2.DownloadedFile .. autoclass:: b2sdk.v2.MtimeUpdatedFile b2-sdk-python-2.8.0/doc/source/api/encryption/000077500000000000000000000000001474454370000211425ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/encryption/setting.rst000066400000000000000000000007211474454370000233510ustar00rootroot00000000000000.. _encryption_setting: Encryption Settings =================== .. autoclass:: b2sdk.v2.EncryptionKey() :no-members: :special-members: __init__ .. autoclass:: b2sdk.v2.UNKNOWN_KEY_ID :no-members: .. autoclass:: b2sdk.v2.EncryptionSetting() :no-members: :special-members: __init__, as_dict .. autoattribute:: b2sdk.v2.SSE_NONE Commonly used "no encryption" setting .. autoattribute:: b2sdk.v2.SSE_B2_AES Commonly used SSE-B2 setting b2-sdk-python-2.8.0/doc/source/api/encryption/types.rst000066400000000000000000000001531474454370000230370ustar00rootroot00000000000000.. _encryption_types: Encryption Types ================ .. automodule:: b2sdk._internal.encryption.types b2-sdk-python-2.8.0/doc/source/api/enums.rst000066400000000000000000000004301474454370000206260ustar00rootroot00000000000000Enums ===== .. autoclass:: b2sdk.v2.MetadataDirectiveMode :inherited-members: .. autoclass:: b2sdk.v2.NewerFileSyncMode :inherited-members: .. autoclass:: b2sdk.v2.CompareVersionMode :inherited-members: .. autoclass:: b2sdk.v2.KeepOrDeleteMode :inherited-members: b2-sdk-python-2.8.0/doc/source/api/exception.rst000066400000000000000000000002501474454370000214750ustar00rootroot00000000000000Exceptions ========== .. todo:: improve documentation of exceptions, automodule -> autoclass? .. automodule:: b2sdk.v2.exception :members: :undoc-members: b2-sdk-python-2.8.0/doc/source/api/file_lock.rst000066400000000000000000000015761474454370000214420ustar00rootroot00000000000000File locks ========== .. autoclass:: b2sdk.v2.LegalHold() :no-members: :special-members: ON, OFF, UNSET, UNKNOWN, is_on, is_off, is_unknown .. autoclass:: b2sdk.v2.FileRetentionSetting() :no-members: :special-members: __init__ .. autoclass:: b2sdk.v2.RetentionMode() :inherited-members: :members: .. autoclass:: b2sdk.v2.BucketRetentionSetting() :no-members: :special-members: __init__ .. autoclass:: b2sdk.v2.RetentionPeriod() :inherited-members: :special-members: __init__ .. autoclass:: b2sdk.v2.FileLockConfiguration() :no-members: :special-members: __init__ .. autoclass:: b2sdk.v2.UNKNOWN_BUCKET_RETENTION() .. autoclass:: b2sdk.v2.UNKNOWN_FILE_LOCK_CONFIGURATION() .. autoclass:: b2sdk.v2.NO_RETENTION_BUCKET_SETTING() .. autoclass:: b2sdk.v2.NO_RETENTION_FILE_SETTING() .. autoclass:: b2sdk.v2.UNKNOWN_FILE_RETENTION_SETTING() b2-sdk-python-2.8.0/doc/source/api/internal/000077500000000000000000000000001474454370000205645ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/internal/b2http.rst000066400000000000000000000002341474454370000225200ustar00rootroot00000000000000:mod:`b2sdk._internal.b2http` -- thin http client wrapper ========================================================= .. automodule:: b2sdk._internal.b2http b2-sdk-python-2.8.0/doc/source/api/internal/cache.rst000066400000000000000000000002711474454370000223610ustar00rootroot00000000000000:mod:`b2sdk._internal.cache` ============================ .. automodule:: b2sdk._internal.cache :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/raw_api.rst000066400000000000000000000003531474454370000227410ustar00rootroot00000000000000:mod:`b2sdk._internal.raw_api` -- B2 raw api wrapper ==================================================== .. automodule:: b2sdk._internal.raw_api :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/raw_simulator.rst000066400000000000000000000004011474454370000242010ustar00rootroot00000000000000:mod:`b2sdk._internal.raw_simulator` -- B2 raw api simulator ============================================================ .. automodule:: b2sdk._internal.raw_simulator :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/requests.rst000066400000000000000000000003001474454370000231620ustar00rootroot00000000000000:mod:`b2sdk._internal.requests` -- modified requests.models.Response class ========================================================================== .. automodule:: b2sdk._internal.requests b2-sdk-python-2.8.0/doc/source/api/internal/scan/000077500000000000000000000000001474454370000215105ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/internal/scan/folder.rst000066400000000000000000000003131474454370000235120ustar00rootroot00000000000000:mod:`b2sdk._internal.scan.folder` ================================== .. automodule:: b2sdk._internal.scan.folder :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/scan/folder_parser.rst000066400000000000000000000003401474454370000250660ustar00rootroot00000000000000:mod:`b2sdk._internal.scan.folder_parser` ========================================= .. automodule:: b2sdk._internal.scan.folder_parser :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/scan/path.rst000066400000000000000000000003051474454370000231740ustar00rootroot00000000000000:mod:`b2sdk._internal.scan.path` ================================ .. automodule:: b2sdk._internal.scan.path :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/scan/policies.rst000066400000000000000000000003211474454370000240450ustar00rootroot00000000000000:mod:`b2sdk._internal.scan.policies` ==================================== .. automodule:: b2sdk._internal.scan.policies :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/scan/scan.rst000066400000000000000000000003051474454370000231640ustar00rootroot00000000000000:mod:`b2sdk._internal.scan.scan` ================================ .. automodule:: b2sdk._internal.scan.scan :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/session.rst000066400000000000000000000003331474454370000230000ustar00rootroot00000000000000:mod:`b2sdk._internal.session` -- B2 Session ============================================ .. automodule:: b2sdk._internal.session :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/stream/000077500000000000000000000000001474454370000220575ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/internal/stream/chained.rst000066400000000000000000000003601474454370000242030ustar00rootroot00000000000000:mod:`b2sdk._internal.stream.chained` ChainedStream =================================================== .. automodule:: b2sdk._internal.stream.chained :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/stream/hashing.rst000066400000000000000000000003621474454370000242330ustar00rootroot00000000000000:mod:`b2sdk._internal.stream.hashing` StreamWithHash ==================================================== .. automodule:: b2sdk._internal.stream.hashing :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/stream/progress.rst000066400000000000000000000004271474454370000244600ustar00rootroot00000000000000:mod:`b2sdk._internal.stream.progress` Streams with progress reporting ====================================================================== .. automodule:: b2sdk._internal.stream.progress :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/stream/range.rst000066400000000000000000000003641474454370000237100ustar00rootroot00000000000000:mod:`b2sdk._internal.stream.range` RangeOfInputStream ====================================================== .. automodule:: b2sdk._internal.stream.range :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/stream/wrapper.rst000066400000000000000000000003601474454370000242700ustar00rootroot00000000000000:mod:`b2sdk._internal.stream.wrapper` StreamWrapper =================================================== .. automodule:: b2sdk._internal.stream.wrapper :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/sync/000077500000000000000000000000001474454370000215405ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/internal/sync/action.rst000066400000000000000000000003131474454370000235440ustar00rootroot00000000000000:mod:`b2sdk._internal.sync.action` ================================== .. automodule:: b2sdk._internal.sync.action :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/sync/exception.rst000066400000000000000000000003241474454370000242670ustar00rootroot00000000000000:mod:`b2sdk._internal.sync.exception` ===================================== .. automodule:: b2sdk._internal.sync.exception :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/sync/policy.rst000066400000000000000000000003131474454370000235660ustar00rootroot00000000000000:mod:`b2sdk._internal.sync.policy` ================================== .. automodule:: b2sdk._internal.sync.policy :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/sync/policy_manager.rst000066400000000000000000000003431474454370000252630ustar00rootroot00000000000000:mod:`b2sdk._internal.sync.policy_manager` ========================================== .. automodule:: b2sdk._internal.sync.policy_manager :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/sync/sync.rst000066400000000000000000000003051474454370000232440ustar00rootroot00000000000000:mod:`b2sdk._internal.sync.sync` ================================ .. automodule:: b2sdk._internal.sync.sync :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/transfer/000077500000000000000000000000001474454370000224105ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/internal/transfer/inbound/000077500000000000000000000000001474454370000240465ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/internal/transfer/inbound/download_manager.rst000066400000000000000000000005011474454370000300750ustar00rootroot00000000000000:mod:`b2sdk._internal.transfer.inbound.download_manager` -- Manager of downloaders ================================================================================== .. automodule:: b2sdk._internal.transfer.inbound.download_manager :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/transfer/inbound/downloader/000077500000000000000000000000001474454370000262045ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/internal/transfer/inbound/downloader/abstract.rst000066400000000000000000000005101474454370000305350ustar00rootroot00000000000000:mod:`b2sdk._internal.transfer.inbound.downloader.abstract` -- Downloader base class ==================================================================================== .. automodule:: b2sdk._internal.transfer.inbound.downloader.abstract :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/transfer/inbound/downloader/parallel.rst000066400000000000000000000005021474454370000305270ustar00rootroot00000000000000:mod:`b2sdk._internal.transfer.inbound.downloader.parallel` -- ParallelTransferer ================================================================================= .. automodule:: b2sdk._internal.transfer.inbound.downloader.parallel :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/transfer/inbound/downloader/simple.rst000066400000000000000000000004701474454370000302300ustar00rootroot00000000000000:mod:`b2sdk._internal.transfer.inbound.downloader.simple` -- SimpleDownloader ============================================================================= .. automodule:: b2sdk._internal.transfer.inbound.downloader.simple :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/transfer/outbound/000077500000000000000000000000001474454370000242475ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/internal/transfer/outbound/upload_source.rst000066400000000000000000000004071474454370000276460ustar00rootroot00000000000000:mod:`b2sdk._internal.transfer.outbound.upload_source` ====================================================== .. automodule:: b2sdk._internal.transfer.outbound.upload_source :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/internal/utils.rst000066400000000000000000000002711474454370000224560ustar00rootroot00000000000000:mod:`b2sdk._internal.utils` ============================ .. automodule:: b2sdk._internal.utils :members: :undoc-members: :show-inheritance: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/progress.rst000066400000000000000000000012031474454370000213420ustar00rootroot00000000000000Progress reporters ================== .. note:: Concrete classes described in this chapter implement methods defined in ``AbstractProgressListener`` .. todo:: improve documentation of progress reporters include info about sync progress .. autoclass:: b2sdk.v2.AbstractProgressListener :inherited-members: :members: .. autoclass:: b2sdk.v2.TqdmProgressListener :no-members: .. autoclass:: b2sdk.v2.SimpleProgressListener :no-members: .. autoclass:: b2sdk.v2.DoNothingProgressListener :no-members: .. autoclass:: b2sdk.v2.ProgressListenerForTest :no-members: .. autofunction:: b2sdk.v2.make_progress_listener b2-sdk-python-2.8.0/doc/source/api/sync.rst000066400000000000000000000203241474454370000204570ustar00rootroot00000000000000.. _sync: ######################## Synchronizer ######################## Synchronizer is a powerful utility with functionality of a basic backup application. It is able to copy entire folders into the cloud and back to a local drive or even between two cloud buckets, providing retention policies and many other options. The **high performance** of sync is credited to parallelization of: * listing local directory contents * listing bucket contents * uploads * downloads Synchronizer spawns threads to perform the operations listed above in parallel to shorten the backup window to a minimum. Sync Options ============ Following are the important optional arguments that can be provided while initializing `Synchronizer` class. * ``compare_version_mode``: When comparing the source and destination files for finding whether to replace them or not, `compare_version_mode` can be passed to specify the mode of comparison. For possible values see :class:`b2sdk.v2.CompareVersionMode`. Default value is :py:attr:`b2sdk.v2.CompareVersionMode.MODTIME` * ``compare_threshold``: It's the minimum size (in bytes)/modification time (in seconds) difference between source and destination files before we assume that it is new and replace. * ``newer_file_mode``: To identify whether to skip or replace if source is older. For possible values see :class:`b2sdk.v2.NewerFileSyncMode`. If you don't specify this the sync will raise :class:`b2sdk.v2.exception.DestFileNewer` in case any of the source file is older than destination. * ``keep_days_or_delete``: specify policy to keep or delete older files. For possible values see :class:`b2sdk.v2.KeepOrDeleteMode`. Default is `DO_NOTHING`. * ``keep_days``: if `keep_days_or_delete` is :py:attr:`b2sdk.v2.KeepOrDeleteMode.KEEP_BEFORE_DELETE` then this specifies for how many days should we keep. .. code-block:: python >>> from b2sdk.v2 import ScanPoliciesManager >>> from b2sdk.v2 import parse_folder >>> from b2sdk.v2 import Synchronizer, SyncReport >>> from b2sdk.v2 import KeepOrDeleteMode, CompareVersionMode, NewerFileSyncMode >>> import time >>> import sys >>> source = '/home/user1/b2_example' >>> destination = 'b2://example-mybucket-b2' >>> source = parse_folder(source, b2_api) >>> destination = parse_folder(destination, b2_api) >>> policies_manager = ScanPoliciesManager(exclude_all_symlinks=True) >>> synchronizer = Synchronizer( max_workers=10, policies_manager=policies_manager, dry_run=False, allow_empty_source=True, compare_version_mode=CompareVersionMode.SIZE, compare_threshold=10, newer_file_mode=NewerFileSyncMode.REPLACE, keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=10, ) We have a file (hello.txt) which is present in destination but not on source (my local), so it will be deleted and since our mode is to keep the delete file, it will be hidden for 10 days in bucket. .. code-block:: python >>> no_progress = False >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) upload f1.txt delete hello.txt (old version) hide hello.txt We changed f1.txt and added 1 byte. Since our compare_threshold is 10, it will not do anything. .. code-block:: python >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) We changed f1.txt and added more than 10 bytes. Since our compare_threshold is 10, it will replace the file at destination folder. .. code-block:: python >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) upload f1.txt Let's just delete the file and not keep - keep_days_or_delete = DELETE You can avoid passing keep_days argument in this case because it will be ignored anyways .. code-block:: python >>> synchronizer = Synchronizer( max_workers=10, policies_manager=policies_manager, dry_run=False, allow_empty_source=True, compare_version_mode=CompareVersionMode.SIZE, compare_threshold=10, # in bytes newer_file_mode=NewerFileSyncMode.REPLACE, keep_days_or_delete=KeepOrDeleteMode.DELETE, ) >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) delete f1.txt delete f1.txt (old version) delete hello.txt (old version) upload f2.txt delete hello.txt (hide marker) As you can see, it deleted f1.txt and it's older versions (no hide this time) and deleted hello.txt also because now we don't want the file anymore. also, we added another file f2.txt which gets uploaded. Now we changed newer_file_mode to SKIP and compare_version_mode to MODTIME. also uploaded a new version of f2.txt to bucket using B2 web. .. code-block:: python >>> synchronizer = Synchronizer( max_workers=10, policies_manager=policies_manager, dry_run=False, allow_empty_source=True, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=10, # in seconds newer_file_mode=NewerFileSyncMode.SKIP, keep_days_or_delete=KeepOrDeleteMode.DELETE, ) >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) As expected, nothing happened, it found a file that was older at source but did not do anything because we skipped. Now we changed newer_file_mode again to REPLACE and also uploaded a new version of f2.txt to bucket using B2 web. .. code-block:: python >>> synchronizer = Synchronizer( max_workers=10, policies_manager=policies_manager, dry_run=False, allow_empty_source=True, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=10, newer_file_mode=NewerFileSyncMode.REPLACE, keep_days_or_delete=KeepOrDeleteMode.DELETE, ) >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, ) delete f2.txt (old version) upload f2.txt Handling encryption ------------------- The `Synchronizer` object may need `EncryptionSetting` instances to perform downloads and copies. For this reason, the `sync_folder` method accepts an `EncryptionSettingsProvider`, see :ref:`server_side_encryption` for further explanation and :ref:`encryption_provider` for public API. Public API classes ================== .. autoclass:: b2sdk.v2.ScanPoliciesManager() :inherited-members: :special-members: __init__ :members: .. autoclass:: b2sdk.v2.Synchronizer() :inherited-members: :special-members: __init__ :members: .. autoclass:: b2sdk.v2.SyncReport() :inherited-members: :special-members: __init__ :members: .. _encryption_provider: Sync Encryption Settings Providers ================================== .. autoclass:: b2sdk.v2.AbstractSyncEncryptionSettingsProvider() :inherited-members: :members: .. autoclass:: b2sdk.v2.ServerDefaultSyncEncryptionSettingsProvider() :no-members: .. autoclass:: b2sdk.v2.BasicSyncEncryptionSettingsProvider() :special-members: __init__ :no-members: b2-sdk-python-2.8.0/doc/source/api/transfer/000077500000000000000000000000001474454370000205745ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/transfer/emerge/000077500000000000000000000000001474454370000220405ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/transfer/emerge/write_intent.rst000066400000000000000000000001701474454370000253030ustar00rootroot00000000000000Write intent ============ .. autoclass:: b2sdk.v2.WriteIntent() :inherited-members: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/transfer/outbound/000077500000000000000000000000001474454370000224335ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/api/transfer/outbound/outbound_source.rst000066400000000000000000000002331474454370000264020ustar00rootroot00000000000000Outbound Transfer Source ======================== .. autoclass:: b2sdk.v2.OutboundTransferSource() :inherited-members: :special-members: __init__ b2-sdk-python-2.8.0/doc/source/api/utils.rst000066400000000000000000000010031474454370000206340ustar00rootroot00000000000000B2 Utility functions ==================== .. autofunction:: b2sdk.v2.b2_url_encode .. autofunction:: b2sdk.v2.b2_url_decode .. autofunction:: b2sdk.v2.choose_part_ranges .. autofunction:: b2sdk.v2.fix_windows_path_limit .. autofunction:: b2sdk.v2.format_and_scale_fraction .. autofunction:: b2sdk.v2.format_and_scale_number .. autofunction:: b2sdk.v2.hex_sha1_of_stream .. autofunction:: b2sdk.v2.hex_sha1_of_bytes .. autoclass:: b2sdk.v2.TempDir() :inherited-members: :special-members: __enter__, __exit__ b2-sdk-python-2.8.0/doc/source/api_reference.rst000066400000000000000000000035721474454370000215270ustar00rootroot00000000000000.. hint:: Use :doc:`quick_start` to quickly jump to examples ######################## API Reference ######################## Interface types ======================= **b2sdk** API is divided into two parts, :ref:`public ` and :ref:`internal `. Please pay attention to which interface type you use. .. tip:: :ref:`Pinning versions ` properly ensures the stability of your application. .. _api_public: Public API ======================== .. toctree:: api/application_key api/account_info api/cache api/api api/exception api/bucket api/file_lock api/data_classes api/downloaded_file api/enums api/progress api/sync api/utils api/transfer/emerge/write_intent api/transfer/outbound/outbound_source api/encryption/setting api/encryption/types .. _api_internal: Internal API ======================== .. note:: See :ref:`Internal interface ` chapter to learn when and how to safely use the Internal API .. toctree:: api/internal/session api/internal/raw_api api/internal/b2http api/internal/requests api/internal/utils api/internal/cache api/internal/stream/chained api/internal/stream/hashing api/internal/stream/progress api/internal/stream/range api/internal/stream/wrapper api/internal/scan/folder_parser api/internal/scan/folder api/internal/scan/path api/internal/scan/policies api/internal/scan/scan api/internal/sync/action api/internal/sync/exception api/internal/sync/policy api/internal/sync/policy_manager api/internal/sync/sync api/internal/transfer/inbound/downloader/abstract api/internal/transfer/inbound/downloader/parallel api/internal/transfer/inbound/downloader/simple api/internal/transfer/inbound/download_manager api/internal/transfer/outbound/upload_source api/internal/raw_simulator b2-sdk-python-2.8.0/doc/source/api_types.rst000066400000000000000000000150101474454370000207230ustar00rootroot00000000000000######################## About API interfaces ######################## .. _semantic_versioning: ******************* Semantic versioning ******************* **b2sdk** follows `Semantic Versioning `_ policy, so in essence the version number is ``MAJOR.MINOR.PATCH`` (for example ``1.2.3``) and: - we increase `MAJOR` version when we make **incompatible** API changes - we increase `MINOR` version when we add functionality **in a backwards-compatible manner**, and - we increase `PATCH` version when we make backwards-compatible **bug fixes** (unless someone relies on the undocumented behavior of a fixed bug) Therefore when setting up **b2sdk** as a dependency, please make sure to match the version appropriately, for example you could put this in your ``requirements.txt`` to make sure your code is compatible with the ``b2sdk`` version your user will get from pypi:: b2sdk>=1.15.0,<2.0.0 .. _interface_versions: ****************** Interface versions ****************** You might notice that the import structure provided in the documentation looks a little odd: ``from b2sdk.v2 import ...``. The ``.v2`` part is used to keep the interface fluid without risk of breaking applications that use the old signatures. With new versions, **b2sdk** will provide functions with signatures matching the old ones, wrapping the new interface in place of the old one. What this means for a developer using **b2sdk**, is that it will just keep working. We have already deleted some legacy functions when moving from ``.v0`` to ``.v1``, providing equivalent wrappers to reduce the migration effort for applications using pre-1.0 versions of **b2sdk** to fixing imports. It also means that **b2sdk** developers may change the interface in the future and will not need to maintain many branches and backport fixes to keep compatibility of for users of those old branches. .. _interface_version_compatibility: ******************************* Interface version compatibility ******************************* A :term:`numbered interface` will not be exactly identical throughout its lifespan, which should not be a problem for anyone, however just in case, the acceptable differences that the developer must tolerate, are listed below. Exceptions ========== The exception hierarchy may change in a backwards compatible manner and the developer must anticipate it. For example, if ``b2sdk.v2.ExceptionC`` inherits directly from ``b2sdk.v2.ExceptionA``, it may one day inherit from ``b2sdk.v2.ExceptionB``, which in turn inherits from ``b2sdk.v2.ExceptionA``. Normally this is not a problem if you use ``isinstance()`` and ``super()`` properly, but your code should not call the constructor of a parent class by directly naming it or it might skip the middle class of the hierarchy (``ExceptionB`` in this example). Extensions ========== Even in the same interface version, objects/classes/enums can get additional fields and their representations such as ``as_dict()`` or ``__repr__`` (but not ``__str__``) may start to contain those fields. Methods and functions can start accepting new **optional** arguments. New methods can be added to existing classes. Performance =========== Some effort will be put into keeping the performance of the old interfaces, but in rare situations old interfaces may end up with a slightly degraded performance after a new version of the library is released. If performance target is absolutely critical to your application, you can pin your dependencies to the middle version (using ``b2sdk>=X.Y.0,` section. This should be used in 99% of use cases, it's enough to implement anything from a `console tool `_ to a `FUSE filesystem `_. Those modules will generally not change in a backwards-incompatible way between non-major versions. Please see :ref:`interface version compatibility ` chapter for notes on what changes must be expected. .. hint:: If the current version of **b2sdk** is ``4.5.6`` and you only use the *public* interface, put this in your ``requirements.txt`` to be safe:: b2sdk>=4.5.6,<5.0.0 .. note:: ``b2sdk.*._something`` and ``b2sdk.*.*._something``, while having a name beginning with an underscore, are **NOT** considered public interface. .. _internal_interface: ****************** Internal interface ****************** Some rarely used features of B2 cloud are not implemented in **b2sdk**. Tracking usage of transactions and transferred data is a good example - if it is required, additional work would need to be put into a specialized internal interface layer to enable accounting and reporting. **b2sdk** maintainers are :ref:`very supportive ` in case someone wants to contribute an additional feature. Please consider adding it to the sdk, so that more people can use it. This way it will also receive our updates, unlike a private implementation which would not receive any updates unless you apply them manually ( but that's a lot of work and we both know it's not going to happen). In practice, an implementation can be either shared or will quickly become outdated. The license of **b2sdk** is very permissive, but when considering whether to keep your patches private or public, please take into consideration the long-term cost of keeping up with a dynamic open-source project and/or the cost of missing the updates, especially those related to performance and reliability (as those are being actively developed in parallel to documentation). Internal interface modules are listed in :ref:`API Internal ` section. .. note:: It is OK for you to use our internal interface (better that than copying our source files!), however, if you do, please pin your dependencies to **middle** version, as backwards-incompatible changes may be introduced in a non-major version. Furthermore, it would be greatly appreciated if an issue was filed for such situations, so that **b2sdk** interface can be improved in a future version in order to avoid strict version pinning. .. hint:: If the current version of **b2sdk** is ``4.5.6`` and you are using the *internal* interface, put this in your requirements.txt:: b2sdk>=4.5.6,<4.6.0 b2-sdk-python-2.8.0/doc/source/conf.py000066400000000000000000000152101474454370000174750ustar00rootroot00000000000000###################################################################### # # File: doc/source/conf.py # # Copyright 2019, Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # -*- coding: utf-8 -*- # # B2 Python SDK documentation build configuration file, created by # sphinx-quickstart on Fri Oct 20 18:2doc/source/conf.py # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import datetime import os import sys sys.path.append(os.path.abspath('../..')) from b2sdk.version import VERSION # noqa: E402 # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx_autodoc_typehints', 'sphinx.ext.intersphinx', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.coverage', 'sphinx.ext.graphviz', 'sphinx.ext.autosummary', 'sphinx.ext.todo', #'sphinxcontrib.fulltoc', # 2019-03-29: unfortunately this doesn't work with sphinx_rtd_theme 'sphinxcontrib.plantuml', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = 'b2-sdk-python' year = datetime.date.today().strftime('%Y') author = 'Backblaze' copyright = f'{year}, {author}' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = VERSION.rsplit('.', 1)[0] # The full version, including alpha/beta/rc tags. release = VERSION # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = os.environ.get('B2_SPHINX_TODO', False) and True # -- Options for HTML output ---------------------------------------------- html_context = { 'display_github': True, # Add 'Edit on Github' link instead of 'View page source' 'github_user': 'Backblaze', 'github_repo': project, 'github_version': 'master', 'conf_py_path': '/doc/source/', 'source_suffix': source_suffix, } # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { 'prev_next_buttons_location': 'both', 'collapse_navigation': True, } autodoc_default_options = { 'member-order': 'bysource', 'exclude-members': '__weakref__, _abc_cache, _abc_negative_cache, _abc_negative_cache_version, _abc_registry, _abc_impl', 'members': True, 'undoc-members': True, } always_document_param_types = True # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ['_static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. # # This is required for the alabaster theme # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars html_sidebars = { '**': [ 'about.html', 'navigation.html', 'relations.html', # needs 'show_related': True theme option to display 'searchbox.html', 'donate.html', ] } # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = 'B2_Python_SDKdoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'B2_Python_SDK.tex', 'B2\\_Python\\_SDK', 'Backblaze', 'manual'), ] # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [(master_doc, 'b2_python_sdk', 'B2 Python SDK Documentation', [author], 1)] # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ( master_doc, 'B2_Python_SDK', 'B2 Python SDK Documentation', author, 'B2_Python_SDK', 'Backblaze Python SDK', 'Miscellaneous', ), ] # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} b2-sdk-python-2.8.0/doc/source/contributing.rst000066400000000000000000000064311474454370000214440ustar00rootroot00000000000000.. _contributors_guide: ######################################### Contributors Guide ######################################### We encourage outside contributors to perform changes on our codebase. Many such changes have been merged already. In order to make it easier to contribute, core developers of this project: * provide guidance (through the issue reporting system) * provide tool assisted code review (through the Pull Request system) * maintain a set of unit tests * maintain a set of integration tests (run with a production cloud) * maintain development automation tools using `nox `_ that can easily: * format the code using `ruff `_ * runs linters to find subtle/potential issues with maintainability * run the test suite on multiple Python versions using `pytest `_ * maintain Continuous Integration (by using GitHub Actions) that: * runs all sorts of linters * checks if the Python distribution can be built * runs all tests on a matrix of 6 versions of Python (including pypy) and 3 operating systems (Linux, Mac OS X and Windows) * checks if the documentation can be built properly * maintain other Continuous Integration tools (coverage tracker) You'll need to have `nox `_ installed: * ``pip install nox`` With ``nox``, you can run different sessions (default are ``lint`` and ``test``): * ``format`` -> Format the code. * ``lint`` -> Run linters. * ``test`` (``test-3.7``, ``test-3.8``, ``test-3.9``, ``test-3.10``) -> Run test suite. * ``cover`` -> Perform coverage analysis. * ``build`` -> Build the distribution. * ``deploy`` -> Deploy the distribution to the PyPi. * ``doc`` -> Build the documentation. * ``doc_cover`` -> Perform coverage analysis for the documentation. For example:: $ nox -s format nox > Running session format nox > Creating virtual environment (virtualenv) using python3.10 in .nox/format ... $ nox -s format nox > Running session format nox > Re-using existing virtual environment at .nox/format. ... $ nox --no-venv -s format nox > Running session format ... Sessions ``test``, ``unit``, and ``integration`` can run on many Python versions, 3.7-3.10 by default. Sessions other than ``test`` use the last given Python version, 3.10 by default. You can change it:: export NOX_PYTHONS=3.7,3.8 With the above setting, session ``test`` will run on Python 3.7 and 3.8, and all other sessions on Python 3.8. Given Python interpreters should be installed in the operating system or via `pyenv `_. Linting ############# To run all available linters:: $ nox -s lint Testing ############# To run all tests on every available Python version:: $ nox -s test To run all tests on a specific version:: $ nox -s test-3.10 To run just unit tests:: $ nox -s unit-3.10 To run just integration tests:: $ export B2_TEST_APPLICATION_KEY=your_app_key $ export B2_TEST_APPLICATION_KEY_ID=your_app_key_id $ nox -s integration-3.10 Documentation ############# To build the documentation and watch for changes (including the source code):: $ nox -s doc To just build the documentation:: $ nox --non-interactive -s doc b2-sdk-python-2.8.0/doc/source/dot/000077500000000000000000000000001474454370000167655ustar00rootroot00000000000000b2-sdk-python-2.8.0/doc/source/dot/sqlite_account_info_schema.dot000066400000000000000000000116021474454370000250450ustar00rootroot00000000000000 digraph G { label = "generated by sadisplay v0.4.9"; fontname = "Bitstream Vera Sans" fontsize = 8 node [ fontname = "Bitstream Vera Sans" fontsize = 8 shape = "plaintext" ] edge [ fontname = "Bitstream Vera Sans" fontsize = 8 ] account [label=<
account
⚪ account_auth_tokenTEXT
⚪ account_idTEXT
⚪ account_id_or_app_key_idTEXT
⚪ allowedTEXT
⚪ api_urlTEXT
⚪ application_keyTEXT
⚪ download_urlTEXT
⚪ minimum_part_sizeINTEGER
⚪ realmTEXT
>] bucket [label=<
bucket
⚪ bucket_idTEXT
⚪ bucket_nameTEXT
>] bucket_upload_url [label=<
bucket_upload_url
⚪ bucket_idTEXT
⚪ upload_auth_tokenTEXT
⚪ upload_urlTEXT
>] update_done [label=<
update_done
⚪ update_numberINTEGER
>] edge [ arrowhead = empty ] edge [ arrowhead = ediamond arrowtail = open ] } b2-sdk-python-2.8.0/doc/source/glossary.rst000066400000000000000000000042641474454370000206020ustar00rootroot00000000000000######## Glossary ######## .. glossary:: absoluteMinimumPartSize The smallest large file part size, as indicated during authorization process by the server (in 2019 it used to be ``5MB``, but the server can set it dynamically) account ID An identifier of the B2 account (not login). Looks like this: ``4ba5845d7aaf``. application key ID Since every :term:`account ID` can have multiple access keys associated with it, the keys need to be distinguished from each other. :term:`application key ID` is an identifier of the access key. There are two types of keys: :term:`master application key` and :term:`non-master application key`. application key The secret associated with an :term:`application key ID`, used to authenticate with the server. Looks like this: ``N2Zug0evLcHDlh_L0Z0AJhiGGdY`` or ``0a1bce5ea463a7e4b090ef5bd6bd82b851928ab2c6`` or ``K0014pbwo1zxcIVMnqSNTfWHReU/O3s`` b2sdk version Looks like this: ``v1.0.0`` or ``1.0.0`` and makes version numbers meaningful. See :ref:`Pinning versions ` for more details. b2sdk interface version Looks like this: ``v2`` or ``b2sdk.v2`` and makes maintaining backward compatibility much easier. See :ref:`interface versions ` for more details. master application key This is the first key you have access to, it is available on the B2 web application. This key has all capabilities, access to all :term:`buckets`, and has no file prefix restrictions or expiration. The :term:`application key ID` of the master application key is equal to :term:`account ID`. non-master application key A key which can have restricted capabilities, can only have access to a certain :term:`bucket` or even to just part of it. See ``_ to learn more. Looks like this: ``0014aa9865d6f0000000000b0`` bucket A container that holds files. You can think of buckets as the top-level folders in your B2 Cloud Storage account. There is no limit to the number of files in a bucket, but there is a limit of 100 buckets per account. See ``_ to learn more. b2-sdk-python-2.8.0/doc/source/index.rst000066400000000000000000000043341474454370000200440ustar00rootroot00000000000000.. todolist:: .. note:: **Event Notifications** feature is now in **Private Preview**. See https://www.backblaze.com/blog/announcing-event-notifications/ for details. ######################################### Overview ######################################### **b2sdk** is a client library for easy access to all of the capabilities of B2 Cloud Storage. `B2 command-line tool `_ is an example of how it can be used to provide command-line access to the B2 service, but there are many possible applications (including `FUSE filesystems `_, storage backend drivers for backup applications etc). ######################################### Why use b2sdk? ######################################### .. todo:: delete doc/source/b2sdk? .. todo:: describe raw_simulator in detail .. todo:: fix list consistency style in "Why use b2sdk?", add links When building an application which uses B2 cloud, it is possible to implement an independent B2 API client, but using **b2sdk** allows for: - reuse of code that is already written, with hundreds of unit tests - use of **Synchronizer**, a high-performance, parallel rsync-like utility - developer-friendly library :ref:`api version policy ` which guards your program against incompatible changes - `B2 integration checklist `_ is passed automatically - **raw_simulator** makes it easy to mock the B2 cloud for unit testing purposes - reporting progress of operations to an object of your choice - exception hierarchy makes it easy to display informative messages to users - interrupted transfers are automatically continued - **b2sdk** has been developed for 3 years before it version 1.0.0 was released. It's stable and mature. ######################################### Documentation index ######################################### .. toctree:: install tutorial quick_start server_side_encryption advanced glossary api_types api_reference contributing ######################################### Indices and tables ######################################### * :ref:`genindex` * :ref:`modindex` * :ref:`search` b2-sdk-python-2.8.0/doc/source/install.rst000066400000000000000000000011651474454370000204020ustar00rootroot00000000000000######################## Installation Guide ######################## Installing as a dependency ========================== **b2sdk** can simply be added to ``requirements.txt`` (or equivalent such as ``setup.py``, ``.pipfile`` etc). In order to properly set a dependency, see :ref:`versioning chapter ` for details. .. note:: The stability of your application depends on correct :ref:`pinning of versions `. Installing a development version ================================ To install **b2sdk**, checkout the repository and run:: pip install b2sdk in your python environment. b2-sdk-python-2.8.0/doc/source/quick_start.rst000066400000000000000000000453211474454370000212670ustar00rootroot00000000000000.. _quick_start: ######################## Quick Start Guide ######################## *********************** Prepare b2sdk *********************** .. code-block:: python >>> from b2sdk.v2 import * >>> info = InMemoryAccountInfo() >>> b2_api = B2Api(info, cache=AuthInfoCache(info)) >>> application_key_id = '4a5b6c7d8e9f' >>> application_key = '001b8e23c26ff6efb941e237deb182b9599a84bef7' >>> b2_api.authorize_account("production", application_key_id, application_key) .. tip:: Get credentials from B2 website *************** Synchronization *************** .. code-block:: python >>> from b2sdk.v2 import ScanPoliciesManager >>> from b2sdk.v2 import parse_folder >>> from b2sdk.v2 import Synchronizer >>> from b2sdk.v2 import SyncReport >>> import time >>> import sys >>> source = '/home/user1/b2_example' >>> destination = 'b2://example-mybucket-b2' >>> source = parse_folder(source, b2_api) >>> destination = parse_folder(destination, b2_api) >>> policies_manager = ScanPoliciesManager(exclude_all_symlinks=True) >>> synchronizer = Synchronizer( max_workers=10, policies_manager=policies_manager, dry_run=False, allow_empty_source=True, ) >>> no_progress = False >>> encryption_settings_provider = BasicSyncEncryptionSettingsProvider({ 'bucket1': EncryptionSettings(mode=EncryptionMode.SSE_B2), 'bucket2': EncryptionSettings( mode=EncryptionMode.SSE_C, key=EncryptionKey(secret=b'VkYp3s6v9y$B&E)H@McQfTjWmZq4t7w!', id='user-generated-key-id') ), 'bucket3': None, }) >>> with SyncReport(sys.stdout, no_progress) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=int(round(time.time() * 1000)), reporter=reporter, encryption_settings_provider=encryption_settings_provider, ) upload some.pdf upload som2.pdf .. tip:: Sync is the preferred way of getting data into and out of B2 cloud, because it can achieve *highest performance* due to parallelization of scanning and data transfer operations. To learn more about sync, see :ref:`sync`. Sync uses an encryption provider. In principle, it's a mapping between file metadata (bucket_name, file_info, etc) and `EncryptionSetting`. The reason for employing such a mapping, rather than a single `EncryptionSetting`, is the fact that users of Sync do not necessarily know up front what files it's going to upload and download. This approach enables using unique keys, or key identifiers, across files. This is covered in greater detail in :ref:`server_side_encryption`. In the example above, Sync will assume `SSE-B2` for all files in `bucket1`, `SSE-C` with the key provided for `bucket2` and rely on bucket default for `bucket3`. Should developers need to provide keys per file (and not per bucket), they need to implement their own :class:`b2sdk.v2.AbstractSyncEncryptionSettingsProvider`. ************** Bucket actions ************** List buckets ============ .. code-block:: python >>> b2_api.list_buckets() [Bucket<346501784642eb3e60980d10,example-mybucket-b2-1,allPublic>] >>> for b in b2_api.list_buckets(): print('%s %-10s %s' % (b.id_, b.type_, b.name)) 346501784642eb3e60980d10 allPublic example-mybucket-b2-1 Create a bucket =============== .. code-block:: python >>> bucket_name = 'example-mybucket-b2-1' # must be unique in B2 (across all accounts!) >>> bucket_type = 'allPublic' # or 'allPrivate' >>> b2_api.create_bucket(bucket_name, bucket_type) Bucket<346501784642eb3e60980d10,example-mybucket-b2-1,allPublic> You can optionally store bucket info, CORS rules and lifecycle rules with the bucket. See :meth:`b2sdk.v2.B2Api.create_bucket`. Delete a bucket =============== .. code-block:: python >>> bucket_name = 'example-mybucket-b2-to-delete' >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> b2_api.delete_bucket(bucket) returns `None` if successful, raises an exception in case of error. Update bucket info ================== .. code-block:: python >>> new_bucket_type = 'allPrivate' >>> bucket_name = 'example-mybucket-b2' >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> new_bucket = bucket.update( bucket_type=new_bucket_type, default_server_side_encryption=EncryptionSetting(mode=EncryptionMode.SSE_B2) ) >>> new_bucket.as_dict() {'accountId': '451862be08d0', 'bucketId': '5485a1682662eb3e60980d10', 'bucketInfo': {}, 'bucketName': 'example-mybucket-b2', 'bucketType': 'allPrivate', 'corsRules': [], 'lifecycleRules': [], 'revision': 3, 'defaultServerSideEncryption': {'isClientAuthorizedToRead': True, 'value': {'algorithm': 'AES256', 'mode': 'SSE-B2'}}}, } For more information see :meth:`b2sdk.v2.Bucket.update`. ************ File actions ************ .. tip:: Sync is the preferred way of getting files into and out of B2 cloud, because it can achieve *highest performance* due to parallelization of scanning and data transfer operations. To learn more about sync, see :ref:`sync`. Use the functions described below only if you *really* need to transfer a single file. Upload file =========== .. code-block:: python >>> local_file_path = '/home/user1/b2_example/new.pdf' >>> b2_file_name = 'dummy_new.pdf' >>> file_info = {'how': 'good-file'} >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> bucket.upload_local_file( local_file=local_file_path, file_name=b2_file_name, file_infos=file_info, ) This will work regardless of the size of the file - ``upload_local_file`` automatically uses large file upload API when necessary. For more information see :meth:`b2sdk.v2.Bucket.upload_local_file`. Upload file encrypted with SSE-C -------------------------------- .. code-block:: python >>> local_file_path = '/home/user1/b2_example/new.pdf' >>> b2_file_name = 'dummy_new.pdf' >>> file_info = {'how': 'good-file'} >>> encryption_setting = EncryptionSetting( mode=EncryptionMode.SSE_C, key=EncryptionKey(secret=b'VkYp3s6v9y$B&E)H@McQfTjWmZq4t7w!', id='user-generated-key-id'), ) >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> bucket.upload_local_file( local_file=local_file_path, file_name=b2_file_name, file_infos=file_info, encryption=encryption_setting, ) Download file ============= .. _download-file-by-id: By id ----- .. code-block:: python >>> from b2sdk.v2 import DoNothingProgressListener >>> local_file_path = '/home/user1/b2_example/new2.pdf' >>> file_id = '4_z5485a1682662eb3e60980d10_f1195145f42952533_d20190403_m130258_c002_v0001111_t0002' >>> progress_listener = DoNothingProgressListener() >>> downloaded_file = b2_api.download_file_by_id(file_id, progress_listener) # only the headers # and the beginning of the file is downloaded at this stage >>> print('File name: ', downloaded_file.download_version.file_name) File name: som2.pdf >>> print('File id: ', downloaded_file.download_version.id_) File id: 4_z5485a1682662eb3e60980d10_f1195145f42952533_d20190403_m130258_c002_v0001111_t0002 >>> print('File size: ', downloaded_file.download_version.size) File size: 1870579 >>> print('Content type:', downloaded_file.download_version.content_type) Content type: application/pdf >>> print('Content sha1:', downloaded_file.download_version.content_sha1) Content sha1: d821849a70922e87c2b0786c0be7266b89d87df0 >>> downloaded_file.save_to(local_file_path) # this downloads the whole file .. _download-file-by-name: By name ------- .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> b2_file_name = 'dummy_new.pdf' >>> local_file_name = '/home/user1/b2_example/new3.pdf' >>> downloaded_file = bucket.download_file_by_name(b2_file_name) >>> downloaded_file.save_to(local_file_path) Downloading encrypted files --------------------------- Both methods (:ref:`download-file-by-name` and :ref:`download-file-by-id`) accept an optional `encryption` argument, similarly to `Upload file`_. This parameter is necessary for downloading files encrypted with `SSE-C`. List files ========== .. code-block:: python >>> bucket_name = 'example-mybucket-b2' >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> for file_version, folder_name in bucket.ls(latest_only=True): >>> print(file_version.file_name, file_version.upload_timestamp, folder_name) f2.txt 1560927489000 None som2.pdf 1554296578000 None some.pdf 1554296579000 None test-folder/.bzEmpty 1561005295000 test-folder/ # Recursive >>> bucket_name = 'example-mybucket-b2' >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> for file_version, folder_name in bucket.ls(latest_only=True, recursive=True): >>> print(file_version.file_name, file_version.upload_timestamp, folder_name) f2.txt 1560927489000 None som2.pdf 1554296578000 None some.pdf 1554296579000 None test-folder/.bzEmpty 1561005295000 test-folder/ test-folder/folder_file.txt 1561005349000 None Note: The files are returned recursively and in order so all files in a folder are printed one after another. The folder_name is returned only for the first file in the folder. .. code-block:: python # Within folder >>> bucket_name = 'example-mybucket-b2' >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> for file_version, folder_name in bucket.ls(folder_to_list='test-folder', latest_only=True): >>> print(file_version.file_name, file_version.upload_timestamp, folder_name) test-folder/.bzEmpty 1561005295000 None test-folder/folder_file.txt 1561005349000 None # list file versions >>> for file_version, folder_name in bucket.ls(latest_only=False): >>> print(file_version.file_name, file_version.upload_timestamp, folder_name) f2.txt 1560927489000 None f2.txt 1560849524000 None som2.pdf 1554296578000 None some.pdf 1554296579000 None For more information see :meth:`b2sdk.v2.Bucket.ls`. Get file metadata ========================= .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> file_version = b2_api.get_file_info(file_id) >>> file_version.as_dict() {'accountId': '451862be08d0', 'action': 'upload', 'bucketId': '5485a1682662eb3e60980d10', 'contentLength': 1870579, 'contentSha1': 'd821849a70922e87c2b0786c0be7266b89d87df0', 'contentType': 'application/pdf', 'fileId': '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', 'fileInfo': {'how': 'good-file', 'sse_c_key_id': 'user-generated-key-id'}, 'fileName': 'dummy_new.pdf', 'uploadTimestamp': 1554361150000, "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-C"}, } Update file lock configuration ============================== .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> file_name = 'dummy.pdf' >>> b2_api.update_file_legal_hold(file_id, file_name, LegalHold.ON) >>> b2_api.update_file_legal_hold( file_id, file_name, FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time() + 100)*1000)) This is low-level file API, for high-level operations see `Direct file operations`_. Copy file ========= .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f118df9ba2c5131e8_d20190619_m065809_c002_v0001126_t0040' >>> new_file_version = bucket.copy(file_id, 'f2_copy.txt') >>> new_file_version.as_dict() {'accountId': '451862be08d0', 'action': 'copy', 'bucketId': '5485a1682662eb3e60980d10', 'contentLength': 124, 'contentSha1': '737637702a0e41dda8b7be79c8db1d369c6eef4a', 'contentType': 'text/plain', 'fileId': '4_z5485a1682662eb3e60980d10_f1022e2320daf707f_d20190620_m122848_c002_v0001123_t0020', 'fileInfo': {'src_last_modified_millis': '1560848707000'}, 'fileName': 'f2_copy.txt', 'uploadTimestamp': 1561033728000, "serverSideEncryption": {"algorithm": "AES256", "mode": "SSE-B2"}} If the ``content length`` is not provided and the file is larger than 5GB, ``copy`` would not succeed and error would be raised. If length is provided, then the file may be copied as a large file. Maximum copy part size can be set by ``max_copy_part_size`` - if not set, it will default to 5GB. If ``max_copy_part_size`` is lower than :term:`absoluteMinimumPartSize`, file would be copied in single request - this may be used to force copy in single request large file that fits in server small file limit. Copying files allows for providing encryption settings for both source and destination files - `SSE-C` encrypted source files cannot be used unless the proper key is provided. If you want to copy just the part of the file, then you can specify the offset and content length: .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f118df9ba2c5131e8_d20190619_m065809_c002_v0001126_t0040' >>> bucket.copy(file_id, 'f2_copy.txt', offset=1024, length=2048) Note that content length is required for offset values other than zero. For more information see :meth:`b2sdk.v2.Bucket.copy`. Delete file =========== .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> file_id_and_name = b2_api.delete_file_version(file_id, 'dummy_new.pdf') >>> file_id_and_name.as_dict() {'action': 'delete', 'fileId': '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', 'fileName': 'dummy_new.pdf'} This is low-level file API, for high-level operations see `Direct file operations`_. Cancel large file uploads ========================= .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> for unfinished_file in bucket.list_unfinished_large_files(): b2_api.cancel_large_file(unfinished_file.file_id, unfinished_file.file_name) ********************** Direct file operations ********************** Methods for manipulating object (file) state mentioned in sections above are low level and useful when users have access to basic information, like file id and name. Many API methods, however, return python objects representing files (:class:`b2sdk.v2.FileVersion` and :class:`b2sdk.v2.DownloadVersion`), that provide high-level access to methods manipulating their state. As a rule, these methods don't change properties of python objects they are called on, but return new objects instead. Obtain file representing objects ================================ :class:`b2sdk.v2.FileVersion` ----------------------------- By id ***** .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> file_version = b2_api.get_file_info(file_id) By listing ********** .. code-block:: python >>> for file_version, folder_name in bucket.ls(latest_only=True, prefix='dir_name'): >>> ... :class:`b2sdk.v2.DownloadVersion` --------------------------------- By id ***** .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> downloaded_file = b2_api.download_file_by_id(file_id) >>> download_version = downloaded_file.download_version By name ******* .. code-block:: python >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> b2_file_name = 'dummy_new.pdf' >>> downloaded_file = bucket.download_file_by_name(b2_file_name) >>> download_version = downloaded_file.download_version Download (only for :class:`b2sdk.v2.FileVersion`) ================================================= .. code-block:: python >>> file_version.download() >>> # equivalent to >>> b2_api.download_file_by_id(file_version.id_) Delete ====== .. code-block:: python >>> file_version.delete() >>> download_version.delete() >>> # equivalent to >>> b2_api.delete_file_version(file_version.id_, file_version.file_name) >>> b2_api.delete_file_version(download_version.id_, download_version.file_name) Update file lock ================ .. code-block:: python >>> file_version.update_legal_hold(LegalHold.ON) >>> download_version.update_legal_hold(LegalHold.ON) >>> file_version.update_retention( FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time() + 100)*1000)) >>> download_version.update_retention( FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time() + 100)*1000)) >>> # equivalent to >>> b2_api.update_file_legal_hold(file_version.id_, file_version.file_name, LegalHold.ON) >>> b2_api.update_file_legal_hold(download_version.id_, download_version.file_name, LegalHold.ON) >>> b2_api.update_file_legal_hold( file_version.id_, file_version.file_name, FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time() + 100)*1000)) >>> b2_api.update_file_legal_hold( download_version.id_, download_version.file_name, FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time() + 100)*1000)) Usage examples ============== .. code-block:: python >>> for file_version, folder_name in bucket.ls(latest_only=True, prefix='dir_name'): >>> if file_version.mod_time_millis < 1627979193913 and file_version.file_name.endswith('.csv'): >>> if file_version.legal_hold.is_on(): >>> file_version = file_version.update_legal_hold(LegalHold.OFF) >>> file_version.delete() >>> else: >>> file_version.download().save_to(Path('/tmp/dir_name') / file_version.file_name) .. code-block:: python >>> file_id = '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044' >>> downloaded_file = b2_api.download_file_by_id(file_id) >>> download_version = downloaded_file.download_version >>> if download_version.content_type == 'video/mp4': >>> downloaded_file.save_to(Path('/tmp/dir_name') / download_version.file_name) >>> if download_version.file_retention != NO_RETENTION_FILE_SETTING: >>> download_version = download_version.update_retention( NO_RETENTION_FILE_SETTING, bypass_governance=True) >>> download_version.delete() b2-sdk-python-2.8.0/doc/source/server_side_encryption.rst000066400000000000000000000063051474454370000235210ustar00rootroot00000000000000.. _server_side_encryption: ######################## Server-Side Encryption ######################## *********************** Cloud *********************** B2 cloud supports `Server-Side Encryption `_. All read and write operations provided by **b2sdk** accept encryption settings as an optional argument. Not supplying this argument means relying on bucket defaults - for **SSE-B2** and for no encryption. In case of **SSE-C**, providing a decryption key is required for successful downloading and copying. *** API *** Methods and classes that require encryption settings all accept an `EncryptionSetting` object, which holds information about present or desired encryption (mode, algorithm, key, key_id). Some, like copy operations, accept two sets of settings (for source and for destination). Sync, however, accepts an `EncryptionSettingsProvider` object, which is an `EncryptionSetting` factory, yielding them based on file metadata. For details, see * :ref:`encryption_setting` * :ref:`encryption_types` * :ref:`encryption_provider` ****************************** High security: use unique keys ****************************** B2 cloud does not promote or discourage either reusing encryption keys or using unique keys for `SSE-C`. In applications requiring enhanced security, using unique key per file is a good strategy. **b2sdk** follows a convention, that makes managing such keys easier: `EncryptionSetting` holds a key identifier, aside from the key itself. This key identifier is saved in the metadata of all files uploaded, created or copied via **b2sdk** methods using `SSE-C`, under `sse_c_key_id` in `fileInfo`. This allows developers to create key managers that map those ids to keys, stored securely in a file or a database. Implementing such managers, and linking them to :class:`b2sdk.v2.AbstractSyncEncryptionSettingsProvider` implementations (necessary for using Sync) is outside of the scope of this library. There is, however, a convention to such managers that authors of this library strongly suggest: if a manager needs to generate a new key-key_id pair for uploading, it's best to commit this data to the underlying storage before commencing the upload. The justification of such approach is: should the key-key_id pair be committed to permanent storage after completing an IO operation, committing could fail after successfully upload the data. This data, however, is now just a random blob, that can never be read, since the key to decrypting it is lost. This approach comes an overhead: to download a file, its `fileInfo` has to be known. This means that fetching metadata is required before downloading. ********************************************* Moderate security: a single key with many ids ********************************************* Should the application's infrastructure require a single key (or a limited set of keys) to be used in a bucket, authors of this library recommend generating a unique key identifier for each file anyway (even though these unique identifiers all point to the same key value). This obfuscates confidential metadata from malicious users, like which files are encrypted with the same key, the total number of different keys, etc. b2-sdk-python-2.8.0/doc/source/tutorial.rst000066400000000000000000000072341474454370000206020ustar00rootroot00000000000000######################################### Tutorial ######################################### *************************** AccountInfo *************************** ``AccountInfo`` object holds information about access keys, tokens, upload urls, as well as a bucket id-name map. It is the first object that you need to create to use **b2sdk**. Using ``AccountInfo``, we'll be able to create a ``B2Api`` object to manage a B2 account. In the tutorial we will use :py:class:`b2sdk.v2.InMemoryAccountInfo`: .. code-block:: python >>> from b2sdk.v2 import InMemoryAccountInfo >>> info = InMemoryAccountInfo() # store credentials, tokens and cache in memory With the ``info`` object in hand, we can now proceed to create a ``B2Api`` object. .. note:: :ref:`AccountInfo` section provides guidance for choosing the correct ``AccountInfo`` class for your application. ********************* Account authorization ********************* .. code-block:: python >>> from b2sdk.v2 import B2Api >>> b2_api = B2Api(info) >>> application_key_id = '4a5b6c7d8e9f' >>> application_key = '001b8e23c26ff6efb941e237deb182b9599a84bef7' >>> b2_api.authorize_account("production", application_key_id, application_key) .. tip:: Get credentials from B2 website To find out more about account authorization, see :meth:`b2sdk.v2.B2Api.authorize_account` *************************** B2Api *************************** *B2Api* allows for account-level operations on a B2 account. Typical B2Api operations ======================== .. currentmodule:: b2sdk.v2.B2Api .. autosummary:: :nosignatures: authorize_account create_bucket delete_bucket list_buckets get_bucket_by_name get_bucket_by_id create_key list_keys delete_key download_file_by_id list_parts cancel_large_file .. code-block:: python >>> b2_api = B2Api(info) to find out more, see :class:`b2sdk.v2.B2Api`. The most practical operation on ``B2Api`` object is :meth:`b2sdk.v2.B2Api.get_bucket_by_name`. *Bucket* allows for operations such as listing a remote bucket or transferring files. *************************** Bucket *************************** Initializing a Bucket ======================== Retrieve an existing Bucket --------------------------- To get a ``Bucket`` object for an existing B2 Bucket: .. code-block:: python >>> b2_api.get_bucket_by_name("example-mybucket-b2-1",) Bucket<346501784642eb3e60980d10,example-mybucket-b2-1,allPublic> Create a new Bucket ------------------------ To create a bucket: .. code-block:: python >>> bucket_name = 'example-mybucket-b2-1' >>> bucket_type = 'allPublic' # or 'allPrivate' >>> b2_api.create_bucket(bucket_name, bucket_type) Bucket<346501784642eb3e60980d10,example-mybucket-b2-1,allPublic> You can optionally store bucket info, CORS rules and lifecycle rules with the bucket. See :meth:`b2sdk.v2.B2Api.create_bucket` for more details. .. note:: Bucket name must be unique in B2 (across all accounts!). Your application should be able to cope with a bucket name collision with another B2 user. Typical Bucket operations ========================= .. currentmodule:: b2sdk.v2.Bucket .. autosummary:: :nosignatures: download_file_by_name upload_local_file upload_bytes ls hide_file delete_file_version get_download_authorization get_download_url update set_type set_info To find out more, see :class:`b2sdk.v2.Bucket`. *************************** Summary *************************** You now know how to use ``AccountInfo``, ``B2Api`` and ``Bucket`` objects. To see examples of some of the methods presented above, visit the :ref:`quick start guide ` section. b2-sdk-python-2.8.0/doc/sqlite_account_info_schema.py000077500000000000000000000021111474454370000226170ustar00rootroot00000000000000###################################################################### # # File: doc/sqlite_account_info_schema.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### """ generates a dot file with SqliteAccountInfo database structure """ from __future__ import annotations import tempfile import operator from sadisplay import describe, render from sqlalchemy import create_engine, MetaData from b2sdk._internal.account_info.sqlite_account_info import SqliteAccountInfo def main(): with tempfile.NamedTemporaryFile() as fp: sqlite_db_name = fp.name SqliteAccountInfo(sqlite_db_name) engine = create_engine('sqlite:///' + sqlite_db_name) meta = MetaData() meta.reflect(bind=engine) tables = set(meta.tables.keys()) desc = describe(map(lambda x: operator.getitem(meta.tables, x), sorted(tables))) print(getattr(render, 'dot')(desc).encode('utf-8')) if __name__ == '__main__': main() b2-sdk-python-2.8.0/noxfile.py000066400000000000000000000316721474454370000161610ustar00rootroot00000000000000###################################################################### # # File: noxfile.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import os import pathlib import platform import re import subprocess import nox UPSTREAM_REPO_URL = 'git@github.com:Backblaze/b2-sdk-python.git' # Required for PDM to use nox's virtualenvs os.environ.update({'PDM_IGNORE_SAVED_PYTHON': '1'}) CI = os.environ.get('CI') is not None NOX_PYTHONS = os.environ.get('NOX_PYTHONS') _NOX_EXTRAS = os.environ.get('NOX_EXTRAS') NOX_EXTRAS = [[]] if _NOX_EXTRAS is None else list(filter(None, [_NOX_EXTRAS.split(',')])) PYTHON_VERSIONS = ( [ 'pypy3.9', 'pypy3.10', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', ] if NOX_PYTHONS is None else NOX_PYTHONS.split(',') ) def _detect_python_nox_id() -> str: major, minor, *_ = platform.python_version_tuple() python_nox_id = f'{major}.{minor}' if platform.python_implementation() == 'PyPy': python_nox_id = f'pypy{python_nox_id}' return python_nox_id if CI and not NOX_PYTHONS: # this is done to allow it to work even if `nox -p` was passed to nox PYTHON_VERSIONS = [_detect_python_nox_id()] print(f'CI job mode; using provided interpreter only; PYTHON_VERSIONS={PYTHON_VERSIONS!r}') PYTHON_DEFAULT_VERSION = PYTHON_VERSIONS[-2] if len(PYTHON_VERSIONS) > 1 else PYTHON_VERSIONS[0] PY_PATHS = ['b2sdk', 'test', 'noxfile.py'] nox.options.reuse_existing_virtualenvs = True nox.options.sessions = [ 'lint', 'test', ] def pdm_install(session: nox.Session, *args: str, dev: bool = True) -> None: # dev dependencies are installed by default prod_args = [] if dev else ['--prod'] group_args = [] for group in args: group_args.extend(['--group', group]) session.run('pdm', 'install', *prod_args, *group_args, external=True) def skip_coverage(python_version: str | None) -> bool: return (python_version or platform.python_implementation()).lower().startswith('pypy') @nox.session(name='format', python=PYTHON_DEFAULT_VERSION) def format_(session): """Lint the code and apply fixes in-place whenever possible.""" pdm_install(session, 'lint') session.run('ruff', 'check', '--fix', *PY_PATHS) session.run('ruff', 'format', *PY_PATHS) # session.run( # 'docformatter', # '--in-place', # '--recursive', # '--wrap-summaries=100', # '--wrap-descriptions=100', # *PY_PATHS, # ) @nox.session(python=PYTHON_DEFAULT_VERSION) def lint(session): """Run linters in readonly mode.""" # We need to install 'doc' group because liccheck needs to inspect it. pdm_install(session, 'doc', 'lint', 'full') session.run('ruff', 'check', *PY_PATHS) session.run('ruff', 'format', *PY_PATHS) # session.run( # 'docformatter', # '--check', # '--recursive', # '--wrap-summaries=100', # '--wrap-descriptions=100', # *PY_PATHS, # ) session.run('pytest', 'test/static') session.run('liccheck', '-s', 'pyproject.toml') # Check if the lockfile is up to date session.run('pdm', 'lock', '--check', external=True) @nox.session(python=PYTHON_VERSIONS) @nox.parametrize('extras', NOX_EXTRAS) def unit(session, extras): """Run unit tests.""" pdm_install(session, 'test', *extras) args = ['--doctest-modules', '-n', 'auto'] if not skip_coverage(session.python): args += ['--cov=b2sdk', '--cov-branch', '--cov-report=xml'] # TODO: Use session.parametrize for apiver session.run('pytest', '--api=v3', *args, *session.posargs, 'test/unit') if not skip_coverage(session.python): args += ['--cov-append'] session.run('pytest', '--api=v2', *args, *session.posargs, 'test/unit') session.run('pytest', '--api=v1', *args, *session.posargs, 'test/unit') session.run('pytest', '--api=v0', *args, *session.posargs, 'test/unit') if not skip_coverage(session.python) and not session.posargs: session.notify('cover') @nox.session(python=PYTHON_VERSIONS) @nox.parametrize('extras', NOX_EXTRAS) def integration(session, extras): """Run integration tests.""" pdm_install(session, 'test', *extras) session.run('pytest', '-s', *session.posargs, 'test/integration') @nox.session(python=PYTHON_DEFAULT_VERSION) def cleanup_old_buckets(session): """Remove buckets from previous test runs.""" pdm_install(session, 'test') session.run('python', '-m', 'test.integration.cleanup_buckets') @nox.session(python=PYTHON_VERSIONS) def test(session): """Run all tests.""" if session.python: session.notify(f'unit-{session.python}') session.notify(f'integration-{session.python}') else: session.notify('unit') session.notify('integration') @nox.session def cover(session): """Perform coverage analysis.""" pdm_install(session, 'test') session.run('coverage', 'report', '--fail-under=75', '--show-missing', '--skip-covered') session.run('coverage', 'erase') @nox.session(python=PYTHON_DEFAULT_VERSION) def build(session): """Build the distribution.""" session.run('pdm', 'build', external=True) # Set outputs for GitHub Actions if CI: with open(os.environ['GITHUB_OUTPUT'], 'a') as github_output: # Path have to be specified with unix style slashes even for windows, # otherwise glob won't find files on windows in action-gh-release. print('asset_path=dist/*', file=github_output) version = os.environ['GITHUB_REF'].replace('refs/tags/v', '') print(f'version={version}', file=github_output) @nox.session(python=PYTHON_DEFAULT_VERSION) def doc(session): """Build the documentation.""" pdm_install(session, 'doc') session.cd('doc') sphinx_args = ['-b', 'html', '-T', '-W', 'source', 'build/html'] session.run('rm', '-rf', 'build', external=True) if not session.interactive: session.run('sphinx-build', *sphinx_args) session.notify('doc_cover') else: sphinx_args[-2:-2] = [ '-E', '--open-browser', '--watch', '../b2sdk', '--ignore', '*.pyc', '--ignore', '*~', ] session.run('sphinx-autobuild', *sphinx_args) @nox.session def doc_cover(session): """Perform coverage analysis for the documentation.""" pdm_install(session, 'doc') session.cd('doc') sphinx_args = ['-b', 'coverage', '-T', '-W', 'source', 'build/coverage'] report_file = 'build/coverage/python.txt' session.run('sphinx-build', *sphinx_args) session.run('cat', report_file, external=True) with open('build/coverage/python.txt') as fd: # If there is no undocumented files, the report should have only 2 lines (header) if sum(1 for _ in fd) != 2: session.error('sphinx coverage has failed') @nox.session(python=PYTHON_DEFAULT_VERSION) def make_release_commit(session): """ Runs `towncrier build`, commits changes, tags, all that is left to do is pushing """ if session.posargs: version = session.posargs[0] else: session.error('Provide -- {release_version} (X.Y.Z - without leading "v")') if not re.match(r'^\d+\.\d+\.\d+$', version): session.error( f'Provided version="{version}". Version must be of the form X.Y.Z where ' f'X, Y and Z are integers' ) local_changes = subprocess.check_output(['git', 'diff', '--stat']) if local_changes: session.error('Uncommitted changes detected') current_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode() if current_branch != 'master': session.log('WARNING: releasing from a branch different than master') pdm_install(session, 'release') session.run('towncrier', 'build', '--yes', '--version', version) session.log( f'CHANGELOG updated, changes ready to commit and push\n' f' git remote add upstream {UPSTREAM_REPO_URL!r} 2>/dev/null || git remote get-url upstream\n' f' git commit -m "release {version}"\n' f' git push upstream {current_branch}\n' f'Wait for a CI workflow to complete successfully, before triggering CD by pushing a tag.\n' f' git tag v{version}\n' f' git push upstream v{version}\n' f'Wait for a CD workflow to complete successfully, indicates the release is done.' ) def load_allowed_change_types( project_toml: pathlib.Path = pathlib.Path('./pyproject.toml'), ) -> set[str]: """ Load the list of allowed change types from the pyproject.toml file. """ import tomllib configuration = tomllib.loads(project_toml.read_text()) return set(entry['directory'] for entry in configuration['tool']['towncrier']['type']) def is_changelog_filename_valid(filename: str, allowed_change_types: set[str]) -> tuple[bool, str]: """ Validates whether the given filename matches our rules. Provides information about why it doesn't match them. """ error_reasons = [] wanted_extension = 'md' try: description, change_type, extension = filename.rsplit('.', maxsplit=2) except ValueError: # Not enough values to unpack. return False, 'Doesn\'t follow the "..md" pattern.' # Check whether the filename ends with .md. if extension != wanted_extension: error_reasons.append(f"Doesn't end with {wanted_extension} extension.") # Check whether the change type is valid. if change_type not in allowed_change_types: error_reasons.append( f"Change type '{change_type}' doesn't match allowed types: {allowed_change_types}." ) # Check whether the description makes sense. try: int(description) except ValueError: if description[0] != '+': error_reasons.append("Doesn't start with a number nor a plus sign.") return len(error_reasons) == 0, ' / '.join(error_reasons) if error_reasons else '' def is_changelog_entry_valid(file_content: str) -> tuple[bool, str]: """ We expect the changelog entry to be a valid sentence in the English language. This includes, but not limits to, providing a capital letter at the start and the full-stop character at the end. Note: to do this "properly", tools like `nltk` and `spacy` should be used. """ error_reasons = [] # Check whether the first character is a capital letter. # Not allowing special characters nor numbers at the very start. if not file_content[0].isalpha() or not file_content[0].isupper(): error_reasons.append('The first character is not a capital letter.') # Check if the last character is a full-stop character. if file_content.strip()[-1] != '.': error_reasons.append('The last character is not a full-stop character.') return len(error_reasons) == 0, ' / '.join(error_reasons) if error_reasons else '' @nox.session(python=PYTHON_DEFAULT_VERSION) def towncrier_check(session): """ Check whether all the entries in the changelog.d follow the expected naming convention as well as some basic rules as to their format. """ expected_non_md_files = {'.gitkeep'} allowed_change_types = load_allowed_change_types() is_error = False for filename in pathlib.Path('./changelog.d/').glob('*'): # If that's an expected file, it's all right. if filename.name in expected_non_md_files: continue # Check whether the file matches the expected pattern. is_valid, error_message = is_changelog_filename_valid(filename.name, allowed_change_types) if not is_valid: session.log(f"File {filename.name} doesn't match the expected pattern: {error_message}") is_error = True continue # Check whether the file isn't too big. if filename.lstat().st_size > 16 * 1024: session.log( f'File {filename.name} content is too big – it should be smaller than 16kB.' ) is_error = True continue # Check whether the file can be loaded as UTF-8 file. try: file_content = filename.read_text(encoding='utf-8') except UnicodeDecodeError: session.log(f'File {filename.name} is not a valid UTF-8 file.') is_error = True continue # Check whether the content of the file is anyhow valid. is_valid, error_message = is_changelog_entry_valid(file_content) if not is_valid: session.log(f'File {filename.name} is not a valid changelog entry: {error_message}') is_error = True continue if is_error: session.error( 'Found errors in the changelog.d directory. Check logs above for more information' ) b2-sdk-python-2.8.0/pdm.lock000066400000000000000000003546721474454370000156050ustar00rootroot00000000000000# This file is @generated by PDM. # It is not intended for manual editing. [metadata] groups = ["default", "doc", "format", "lint", "release", "test"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.5.0" content_hash = "sha256:c5cfe2bc633c343a34992598f2b231b43aabcba2ec63e5cdff28cb67c76a0d91" [[metadata.targets]] requires_python = ">=3.8" [[package]] name = "alabaster" version = "0.7.13" requires_python = ">=3.6" summary = "A configurable sidebar-enabled Sphinx theme" groups = ["doc"] files = [ {file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"}, {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, ] [[package]] name = "annotated-types" version = "0.7.0" requires_python = ">=3.8" summary = "Reusable constraint types to use with typing.Annotated" groups = ["default", "test"] dependencies = [ "typing-extensions>=4.0.0; python_version < \"3.9\"", ] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, ] [[package]] name = "babel" version = "2.16.0" requires_python = ">=3.8" summary = "Internationalization utilities" groups = ["doc"] dependencies = [ "pytz>=2015.7; python_version < \"3.9\"", ] files = [ {file = "babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b"}, {file = "babel-2.16.0.tar.gz", hash = "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316"}, ] [[package]] name = "certifi" version = "2024.12.14" requires_python = ">=3.6" summary = "Python package for providing Mozilla's CA Bundle." groups = ["default", "doc"] files = [ {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, ] [[package]] name = "charset-normalizer" version = "3.4.0" requires_python = ">=3.7.0" summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." groups = ["default", "doc"] files = [ {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, ] [[package]] name = "click" version = "8.1.7" requires_python = ">=3.7" summary = "Composable command line interface toolkit" groups = ["release"] marker = "python_version >= \"3.8\"" dependencies = [ "colorama; platform_system == \"Windows\"", "importlib-metadata; python_version < \"3.8\"", ] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [[package]] name = "colorama" version = "0.4.6" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" summary = "Cross-platform colored terminal text." groups = ["doc", "lint", "release", "test"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] [[package]] name = "coverage" version = "7.6.1" requires_python = ">=3.8" summary = "Code coverage measurement for Python" groups = ["test"] files = [ {file = "coverage-7.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16"}, {file = "coverage-7.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36"}, {file = "coverage-7.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02"}, {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc"}, {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23"}, {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34"}, {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c"}, {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959"}, {file = "coverage-7.6.1-cp310-cp310-win32.whl", hash = "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232"}, {file = "coverage-7.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0"}, {file = "coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93"}, {file = "coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3"}, {file = "coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff"}, {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d"}, {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6"}, {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56"}, {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234"}, {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133"}, {file = "coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c"}, {file = "coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6"}, {file = "coverage-7.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778"}, {file = "coverage-7.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391"}, {file = "coverage-7.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8"}, {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d"}, {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca"}, {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163"}, {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a"}, {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d"}, {file = "coverage-7.6.1-cp312-cp312-win32.whl", hash = "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5"}, {file = "coverage-7.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb"}, {file = "coverage-7.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106"}, {file = "coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9"}, {file = "coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c"}, {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a"}, {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060"}, {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862"}, {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388"}, {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155"}, {file = "coverage-7.6.1-cp313-cp313-win32.whl", hash = "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a"}, {file = "coverage-7.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129"}, {file = "coverage-7.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e"}, {file = "coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962"}, {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb"}, {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704"}, {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b"}, {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f"}, {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223"}, {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3"}, {file = "coverage-7.6.1-cp313-cp313t-win32.whl", hash = "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f"}, {file = "coverage-7.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657"}, {file = "coverage-7.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0"}, {file = "coverage-7.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a"}, {file = "coverage-7.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b"}, {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3"}, {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de"}, {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6"}, {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569"}, {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989"}, {file = "coverage-7.6.1-cp38-cp38-win32.whl", hash = "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7"}, {file = "coverage-7.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8"}, {file = "coverage-7.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255"}, {file = "coverage-7.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8"}, {file = "coverage-7.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2"}, {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a"}, {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc"}, {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004"}, {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb"}, {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36"}, {file = "coverage-7.6.1-cp39-cp39-win32.whl", hash = "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c"}, {file = "coverage-7.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca"}, {file = "coverage-7.6.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df"}, {file = "coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d"}, ] [[package]] name = "coverage" version = "7.6.1" extras = ["toml"] requires_python = ">=3.8" summary = "Code coverage measurement for Python" groups = ["test"] dependencies = [ "coverage==7.6.1", "tomli; python_full_version <= \"3.11.0a6\"", ] files = [ {file = "coverage-7.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16"}, {file = "coverage-7.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36"}, {file = "coverage-7.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02"}, {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc"}, {file = "coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23"}, {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34"}, {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c"}, {file = "coverage-7.6.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959"}, {file = "coverage-7.6.1-cp310-cp310-win32.whl", hash = "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232"}, {file = "coverage-7.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0"}, {file = "coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93"}, {file = "coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3"}, {file = "coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff"}, {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d"}, {file = "coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6"}, {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56"}, {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234"}, {file = "coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133"}, {file = "coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c"}, {file = "coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6"}, {file = "coverage-7.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778"}, {file = "coverage-7.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391"}, {file = "coverage-7.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8"}, {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d"}, {file = "coverage-7.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca"}, {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163"}, {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a"}, {file = "coverage-7.6.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d"}, {file = "coverage-7.6.1-cp312-cp312-win32.whl", hash = "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5"}, {file = "coverage-7.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb"}, {file = "coverage-7.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106"}, {file = "coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9"}, {file = "coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c"}, {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a"}, {file = "coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060"}, {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862"}, {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388"}, {file = "coverage-7.6.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155"}, {file = "coverage-7.6.1-cp313-cp313-win32.whl", hash = "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a"}, {file = "coverage-7.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129"}, {file = "coverage-7.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e"}, {file = "coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962"}, {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb"}, {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704"}, {file = "coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b"}, {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f"}, {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223"}, {file = "coverage-7.6.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3"}, {file = "coverage-7.6.1-cp313-cp313t-win32.whl", hash = "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f"}, {file = "coverage-7.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657"}, {file = "coverage-7.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0"}, {file = "coverage-7.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a"}, {file = "coverage-7.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b"}, {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3"}, {file = "coverage-7.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de"}, {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6"}, {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569"}, {file = "coverage-7.6.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989"}, {file = "coverage-7.6.1-cp38-cp38-win32.whl", hash = "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7"}, {file = "coverage-7.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8"}, {file = "coverage-7.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255"}, {file = "coverage-7.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8"}, {file = "coverage-7.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2"}, {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a"}, {file = "coverage-7.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc"}, {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004"}, {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb"}, {file = "coverage-7.6.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36"}, {file = "coverage-7.6.1-cp39-cp39-win32.whl", hash = "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c"}, {file = "coverage-7.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca"}, {file = "coverage-7.6.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df"}, {file = "coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d"}, ] [[package]] name = "docutils" version = "0.19" requires_python = ">=3.7" summary = "Docutils -- Python Documentation Utilities" groups = ["doc"] files = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] [[package]] name = "eval-type-backport" version = "0.2.0" requires_python = ">=3.8" summary = "Like `typing._eval_type`, but lets older Python versions use newer typing features." groups = ["test"] marker = "python_version < \"3.10\"" files = [ {file = "eval_type_backport-0.2.0-py3-none-any.whl", hash = "sha256:ac2f73d30d40c5a30a80b8739a789d6bb5e49fdffa66d7912667e2015d9c9933"}, {file = "eval_type_backport-0.2.0.tar.gz", hash = "sha256:68796cfbc7371ebf923f03bdf7bef415f3ec098aeced24e054b253a0e78f7b37"}, ] [[package]] name = "exceptiongroup" version = "1.2.2" requires_python = ">=3.7" summary = "Backport of PEP 654 (exception groups)" groups = ["lint", "test"] marker = "python_version < \"3.11\"" files = [ {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, ] [[package]] name = "execnet" version = "2.1.1" requires_python = ">=3.8" summary = "execnet: rapid multi-Python deployment" groups = ["test"] files = [ {file = "execnet-2.1.1-py3-none-any.whl", hash = "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc"}, {file = "execnet-2.1.1.tar.gz", hash = "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3"}, ] [[package]] name = "greenlet" version = "3.1.1" requires_python = ">=3.7" summary = "Lightweight in-process concurrent programming" groups = ["doc"] marker = "(platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\") and python_version < \"3.13\"" files = [ {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0"}, {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120"}, {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc"}, {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617"}, {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7"}, {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6"}, {file = "greenlet-3.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80"}, {file = "greenlet-3.1.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70"}, {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159"}, {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e"}, {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1"}, {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383"}, {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a"}, {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511"}, {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395"}, {file = "greenlet-3.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39"}, {file = "greenlet-3.1.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d"}, {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79"}, {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa"}, {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441"}, {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36"}, {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9"}, {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0"}, {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942"}, {file = "greenlet-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01"}, {file = "greenlet-3.1.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1"}, {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff"}, {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a"}, {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e"}, {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4"}, {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e"}, {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1"}, {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c"}, {file = "greenlet-3.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761"}, {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011"}, {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13"}, {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475"}, {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b"}, {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822"}, {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01"}, {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6"}, {file = "greenlet-3.1.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:346bed03fe47414091be4ad44786d1bd8bef0c3fcad6ed3dee074a032ab408a9"}, {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfc59d69fc48664bc693842bd57acfdd490acafda1ab52c7836e3fc75c90a111"}, {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21e10da6ec19b457b82636209cbe2331ff4306b54d06fa04b7c138ba18c8a81"}, {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37b9de5a96111fc15418819ab4c4432e4f3c2ede61e660b1e33971eba26ef9ba"}, {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef9ea3f137e5711f0dbe5f9263e8c009b7069d8a1acea822bd5e9dae0ae49c8"}, {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85f3ff71e2e60bd4b4932a043fbbe0f499e263c628390b285cb599154a3b03b1"}, {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:95ffcf719966dd7c453f908e208e14cde192e09fde6c7186c8f1896ef778d8cd"}, {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7"}, {file = "greenlet-3.1.1-cp38-cp38-win32.whl", hash = "sha256:8b8b36671f10ba80e159378df9c4f15c14098c4fd73a36b9ad715f057272fbef"}, {file = "greenlet-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:7017b2be767b9d43cc31416aba48aab0d2309ee31b4dbf10a1d38fb7972bdf9d"}, {file = "greenlet-3.1.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3"}, {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42"}, {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"}, {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437"}, {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145"}, {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c"}, {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e"}, {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e"}, {file = "greenlet-3.1.1-cp39-cp39-win32.whl", hash = "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c"}, {file = "greenlet-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22"}, {file = "greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467"}, ] [[package]] name = "idna" version = "3.10" requires_python = ">=3.6" summary = "Internationalized Domain Names in Applications (IDNA)" groups = ["default", "doc"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, ] [[package]] name = "imagesize" version = "1.4.1" requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" summary = "Getting image size from png/jpeg/jpeg2000/gif file" groups = ["doc"] files = [ {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, ] [[package]] name = "importlib-metadata" version = "8.5.0" requires_python = ">=3.8" summary = "Read metadata from Python packages" groups = ["doc"] marker = "python_version < \"3.10\"" dependencies = [ "typing-extensions>=3.6.4; python_version < \"3.8\"", "zipp>=3.20", ] files = [ {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"}, {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"}, ] [[package]] name = "importlib-resources" version = "6.4.5" requires_python = ">=3.8" summary = "Read resources from Python packages" groups = ["release"] marker = "python_version < \"3.10\" and python_version >= \"3.8\"" dependencies = [ "zipp>=3.1.0; python_version < \"3.10\"", ] files = [ {file = "importlib_resources-6.4.5-py3-none-any.whl", hash = "sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717"}, {file = "importlib_resources-6.4.5.tar.gz", hash = "sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065"}, ] [[package]] name = "incremental" version = "24.7.2" requires_python = ">=3.8" summary = "A small library that versions your Python projects." groups = ["release"] marker = "python_version >= \"3.8\"" dependencies = [ "setuptools>=61.0", "tomli; python_version < \"3.11\"", ] files = [ {file = "incremental-24.7.2-py3-none-any.whl", hash = "sha256:8cb2c3431530bec48ad70513931a760f446ad6c25e8333ca5d95e24b0ed7b8fe"}, {file = "incremental-24.7.2.tar.gz", hash = "sha256:fb4f1d47ee60efe87d4f6f0ebb5f70b9760db2b2574c59c8e8912be4ebd464c9"}, ] [[package]] name = "iniconfig" version = "2.0.0" requires_python = ">=3.7" summary = "brain-dead simple config-ini parsing" groups = ["lint", "test"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] [[package]] name = "jinja2" version = "3.1.4" requires_python = ">=3.7" summary = "A very fast and expressive template engine." groups = ["doc", "release"] dependencies = [ "MarkupSafe>=2.0", ] files = [ {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, ] [[package]] name = "liccheck" version = "0.9.2" requires_python = ">=3.5" summary = "Check python packages from requirement.txt and report issues" groups = ["lint"] dependencies = [ "semantic-version>=2.7.0", "toml", ] files = [ {file = "liccheck-0.9.2-py2.py3-none-any.whl", hash = "sha256:15cbedd042515945fe9d58b62e0a5af2f2a7795def216f163bb35b3016a16637"}, {file = "liccheck-0.9.2.tar.gz", hash = "sha256:bdc2190f8e95af3c8f9c19edb784ba7d41ecb2bf9189422eae6112bf84c08cd5"}, ] [[package]] name = "livereload" version = "2.7.1" requires_python = ">=3.7" summary = "Python LiveReload is an awesome tool for web developers" groups = ["doc"] dependencies = [ "tornado", ] files = [ {file = "livereload-2.7.1-py3-none-any.whl", hash = "sha256:5201740078c1b9433f4b2ba22cd2729a39b9d0ec0a2cc6b4d3df257df5ad0564"}, {file = "livereload-2.7.1.tar.gz", hash = "sha256:3d9bf7c05673df06e32bea23b494b8d36ca6d10f7d5c3c8a6989608c09c986a9"}, ] [[package]] name = "logfury" version = "1.0.1" summary = "('Toolkit for responsible, low-boilerplate logging of library method calls',)" groups = ["default"] files = [ {file = "logfury-1.0.1-py3-none-any.whl", hash = "sha256:b4f04be1701a1df644afc3384d6167d64c899f8036b7eefc3b6c570c6a9b290b"}, {file = "logfury-1.0.1.tar.gz", hash = "sha256:130a5daceab9ad534924252ddf70482aa2c96662b3a3825a7d30981d03b76a26"}, ] [[package]] name = "markupsafe" version = "2.1.5" requires_python = ">=3.7" summary = "Safely add untrusted strings to HTML/XML markup." groups = ["doc", "release"] files = [ {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] [[package]] name = "packaging" version = "24.2" requires_python = ">=3.8" summary = "Core utilities for Python packages" groups = ["doc", "lint", "test"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] [[package]] name = "pluggy" version = "1.5.0" requires_python = ">=3.8" summary = "plugin and hook calling mechanisms for python" groups = ["lint", "test"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [[package]] name = "pydantic" version = "2.10.4" requires_python = ">=3.8" summary = "Data validation using Python type hints" groups = ["test"] dependencies = [ "annotated-types>=0.6.0", "pydantic-core==2.27.2", "typing-extensions>=4.12.2", ] files = [ {file = "pydantic-2.10.4-py3-none-any.whl", hash = "sha256:597e135ea68be3a37552fb524bc7d0d66dcf93d395acd93a00682f1efcb8ee3d"}, {file = "pydantic-2.10.4.tar.gz", hash = "sha256:82f12e9723da6de4fe2ba888b5971157b3be7ad914267dea8f05f82b28254f06"}, ] [[package]] name = "pydantic-core" version = "2.27.2" requires_python = ">=3.8" summary = "Core functionality for Pydantic validation and serialization" groups = ["test"] dependencies = [ "typing-extensions!=4.7.0,>=4.6.0", ] files = [ {file = "pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa"}, {file = "pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c"}, {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7969e133a6f183be60e9f6f56bfae753585680f3b7307a8e555a948d443cc05a"}, {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3de9961f2a346257caf0aa508a4da705467f53778e9ef6fe744c038119737ef5"}, {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2bb4d3e5873c37bb3dd58714d4cd0b0e6238cebc4177ac8fe878f8b3aa8e74c"}, {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:280d219beebb0752699480fe8f1dc61ab6615c2046d76b7ab7ee38858de0a4e7"}, {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47956ae78b6422cbd46f772f1746799cbb862de838fd8d1fbd34a82e05b0983a"}, {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:14d4a5c49d2f009d62a2a7140d3064f686d17a5d1a268bc641954ba181880236"}, {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:337b443af21d488716f8d0b6164de833e788aa6bd7e3a39c005febc1284f4962"}, {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:03d0f86ea3184a12f41a2d23f7ccb79cdb5a18e06993f8a45baa8dfec746f0e9"}, {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7041c36f5680c6e0f08d922aed302e98b3745d97fe1589db0a3eebf6624523af"}, {file = "pydantic_core-2.27.2-cp310-cp310-win32.whl", hash = "sha256:50a68f3e3819077be2c98110c1f9dcb3817e93f267ba80a2c05bb4f8799e2ff4"}, {file = "pydantic_core-2.27.2-cp310-cp310-win_amd64.whl", hash = "sha256:e0fd26b16394ead34a424eecf8a31a1f5137094cabe84a1bcb10fa6ba39d3d31"}, {file = "pydantic_core-2.27.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8e10c99ef58cfdf2a66fc15d66b16c4a04f62bca39db589ae8cba08bc55331bc"}, {file = "pydantic_core-2.27.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26f32e0adf166a84d0cb63be85c562ca8a6fa8de28e5f0d92250c6b7e9e2aff7"}, {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c19d1ea0673cd13cc2f872f6c9ab42acc4e4f492a7ca9d3795ce2b112dd7e15"}, {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e68c4446fe0810e959cdff46ab0a41ce2f2c86d227d96dc3847af0ba7def306"}, {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9640b0059ff4f14d1f37321b94061c6db164fbe49b334b31643e0528d100d99"}, {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40d02e7d45c9f8af700f3452f329ead92da4c5f4317ca9b896de7ce7199ea459"}, {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c1fd185014191700554795c99b347d64f2bb637966c4cfc16998a0ca700d048"}, {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d81d2068e1c1228a565af076598f9e7451712700b673de8f502f0334f281387d"}, {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a4207639fb02ec2dbb76227d7c751a20b1a6b4bc52850568e52260cae64ca3b"}, {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:3de3ce3c9ddc8bbd88f6e0e304dea0e66d843ec9de1b0042b0911c1663ffd474"}, {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:30c5f68ded0c36466acede341551106821043e9afaad516adfb6e8fa80a4e6a6"}, {file = "pydantic_core-2.27.2-cp311-cp311-win32.whl", hash = "sha256:c70c26d2c99f78b125a3459f8afe1aed4d9687c24fd677c6a4436bc042e50d6c"}, {file = "pydantic_core-2.27.2-cp311-cp311-win_amd64.whl", hash = "sha256:08e125dbdc505fa69ca7d9c499639ab6407cfa909214d500897d02afb816e7cc"}, {file = "pydantic_core-2.27.2-cp311-cp311-win_arm64.whl", hash = "sha256:26f0d68d4b235a2bae0c3fc585c585b4ecc51382db0e3ba402a22cbc440915e4"}, {file = "pydantic_core-2.27.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9e0c8cfefa0ef83b4da9588448b6d8d2a2bf1a53c3f1ae5fca39eb3061e2f0b0"}, {file = "pydantic_core-2.27.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83097677b8e3bd7eaa6775720ec8e0405f1575015a463285a92bfdfe254529ef"}, {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:172fce187655fece0c90d90a678424b013f8fbb0ca8b036ac266749c09438cb7"}, {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:519f29f5213271eeeeb3093f662ba2fd512b91c5f188f3bb7b27bc5973816934"}, {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05e3a55d124407fffba0dd6b0c0cd056d10e983ceb4e5dbd10dda135c31071d6"}, {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c3ed807c7b91de05e63930188f19e921d1fe90de6b4f5cd43ee7fcc3525cb8c"}, {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fb4aadc0b9a0c063206846d603b92030eb6f03069151a625667f982887153e2"}, {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28ccb213807e037460326424ceb8b5245acb88f32f3d2777427476e1b32c48c4"}, {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de3cd1899e2c279b140adde9357c4495ed9d47131b4a4eaff9052f23398076b3"}, {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:220f892729375e2d736b97d0e51466252ad84c51857d4d15f5e9692f9ef12be4"}, {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a0fcd29cd6b4e74fe8ddd2c90330fd8edf2e30cb52acda47f06dd615ae72da57"}, {file = "pydantic_core-2.27.2-cp312-cp312-win32.whl", hash = "sha256:1e2cb691ed9834cd6a8be61228471d0a503731abfb42f82458ff27be7b2186fc"}, {file = "pydantic_core-2.27.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc3f1a99a4f4f9dd1de4fe0312c114e740b5ddead65bb4102884b384c15d8bc9"}, {file = "pydantic_core-2.27.2-cp312-cp312-win_arm64.whl", hash = "sha256:3911ac9284cd8a1792d3cb26a2da18f3ca26c6908cc434a18f730dc0db7bfa3b"}, {file = "pydantic_core-2.27.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d14bd329640e63852364c306f4d23eb744e0f8193148d4044dd3dacdaacbd8b"}, {file = "pydantic_core-2.27.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82f91663004eb8ed30ff478d77c4d1179b3563df6cdb15c0817cd1cdaf34d154"}, {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71b24c7d61131bb83df10cc7e687433609963a944ccf45190cfc21e0887b08c9"}, {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa8e459d4954f608fa26116118bb67f56b93b209c39b008277ace29937453dc9"}, {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce8918cbebc8da707ba805b7fd0b382816858728ae7fe19a942080c24e5b7cd1"}, {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3f5c2a021bbc5d976107bb302e0131351c2ba54343f8a496dc8783d3d3a6a"}, {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8086fa684c4775c27f03f062cbb9eaa6e17f064307e86b21b9e0abc9c0f02e"}, {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8d9b3388db186ba0c099a6d20f0604a44eabdeef1777ddd94786cdae158729e4"}, {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7a66efda2387de898c8f38c0cf7f14fca0b51a8ef0b24bfea5849f1b3c95af27"}, {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:18a101c168e4e092ab40dbc2503bdc0f62010e95d292b27827871dc85450d7ee"}, {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ba5dd002f88b78a4215ed2f8ddbdf85e8513382820ba15ad5ad8955ce0ca19a1"}, {file = "pydantic_core-2.27.2-cp313-cp313-win32.whl", hash = "sha256:1ebaf1d0481914d004a573394f4be3a7616334be70261007e47c2a6fe7e50130"}, {file = "pydantic_core-2.27.2-cp313-cp313-win_amd64.whl", hash = "sha256:953101387ecf2f5652883208769a79e48db18c6df442568a0b5ccd8c2723abee"}, {file = "pydantic_core-2.27.2-cp313-cp313-win_arm64.whl", hash = "sha256:ac4dbfd1691affb8f48c2c13241a2e3b60ff23247cbcf981759c768b6633cf8b"}, {file = "pydantic_core-2.27.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d3e8d504bdd3f10835468f29008d72fc8359d95c9c415ce6e767203db6127506"}, {file = "pydantic_core-2.27.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:521eb9b7f036c9b6187f0b47318ab0d7ca14bd87f776240b90b21c1f4f149320"}, {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85210c4d99a0114f5a9481b44560d7d1e35e32cc5634c656bc48e590b669b145"}, {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d716e2e30c6f140d7560ef1538953a5cd1a87264c737643d481f2779fc247fe1"}, {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f66d89ba397d92f840f8654756196d93804278457b5fbede59598a1f9f90b228"}, {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:669e193c1c576a58f132e3158f9dfa9662969edb1a250c54d8fa52590045f046"}, {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdbe7629b996647b99c01b37f11170a57ae675375b14b8c13b8518b8320ced5"}, {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d262606bf386a5ba0b0af3b97f37c83d7011439e3dc1a9298f21efb292e42f1a"}, {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cabb9bcb7e0d97f74df8646f34fc76fbf793b7f6dc2438517d7a9e50eee4f14d"}, {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:d2d63f1215638d28221f664596b1ccb3944f6e25dd18cd3b86b0a4c408d5ebb9"}, {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bca101c00bff0adb45a833f8451b9105d9df18accb8743b08107d7ada14bd7da"}, {file = "pydantic_core-2.27.2-cp38-cp38-win32.whl", hash = "sha256:f6f8e111843bbb0dee4cb6594cdc73e79b3329b526037ec242a3e49012495b3b"}, {file = "pydantic_core-2.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:fd1aea04935a508f62e0d0ef1f5ae968774a32afc306fb8545e06f5ff5cdf3ad"}, {file = "pydantic_core-2.27.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c10eb4f1659290b523af58fa7cffb452a61ad6ae5613404519aee4bfbf1df993"}, {file = "pydantic_core-2.27.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef592d4bad47296fb11f96cd7dc898b92e795032b4894dfb4076cfccd43a9308"}, {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c61709a844acc6bf0b7dce7daae75195a10aac96a596ea1b776996414791ede4"}, {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c5f762659e47fdb7b16956c71598292f60a03aa92f8b6351504359dbdba6cf"}, {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c9775e339e42e79ec99c441d9730fccf07414af63eac2f0e48e08fd38a64d76"}, {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57762139821c31847cfb2df63c12f725788bd9f04bc2fb392790959b8f70f118"}, {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d1e85068e818c73e048fe28cfc769040bb1f475524f4745a5dc621f75ac7630"}, {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:097830ed52fd9e427942ff3b9bc17fab52913b2f50f2880dc4a5611446606a54"}, {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044a50963a614ecfae59bb1eaf7ea7efc4bc62f49ed594e18fa1e5d953c40e9f"}, {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:4e0b4220ba5b40d727c7f879eac379b822eee5d8fff418e9d3381ee45b3b0362"}, {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e4f4bb20d75e9325cc9696c6802657b58bc1dbbe3022f32cc2b2b632c3fbb96"}, {file = "pydantic_core-2.27.2-cp39-cp39-win32.whl", hash = "sha256:cca63613e90d001b9f2f9a9ceb276c308bfa2a43fafb75c8031c4f66039e8c6e"}, {file = "pydantic_core-2.27.2-cp39-cp39-win_amd64.whl", hash = "sha256:77d1bca19b0f7021b3a982e6f903dcd5b2b06076def36a652e3907f596e29f67"}, {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2bf14caea37e91198329b828eae1618c068dfb8ef17bb33287a7ad4b61ac314e"}, {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0cb791f5b45307caae8810c2023a184c74605ec3bcbb67d13846c28ff731ff8"}, {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:688d3fd9fcb71f41c4c015c023d12a79d1c4c0732ec9eb35d96e3388a120dcf3"}, {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d591580c34f4d731592f0e9fe40f9cc1b430d297eecc70b962e93c5c668f15f"}, {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:82f986faf4e644ffc189a7f1aafc86e46ef70372bb153e7001e8afccc6e54133"}, {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bec317a27290e2537f922639cafd54990551725fc844249e64c523301d0822fc"}, {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:0296abcb83a797db256b773f45773da397da75a08f5fcaef41f2044adec05f50"}, {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0d75070718e369e452075a6017fbf187f788e17ed67a3abd47fa934d001863d9"}, {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7e17b560be3c98a8e3aa66ce828bdebb9e9ac6ad5466fba92eb74c4c95cb1151"}, {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c33939a82924da9ed65dab5a65d427205a73181d8098e79b6b426bdf8ad4e656"}, {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:00bad2484fa6bda1e216e7345a798bd37c68fb2d97558edd584942aa41b7d278"}, {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c817e2b40aba42bac6f457498dacabc568c3b7a986fc9ba7c8d9d260b71485fb"}, {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:251136cdad0cb722e93732cb45ca5299fb56e1344a833640bf93b2803f8d1bfd"}, {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d2088237af596f0a524d3afc39ab3b036e8adb054ee57cbb1dcf8e09da5b29cc"}, {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d4041c0b966a84b4ae7a09832eb691a35aec90910cd2dbe7a208de59be77965b"}, {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8083d4e875ebe0b864ffef72a4304827015cff328a1be6e22cc850753bfb122b"}, {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f141ee28a0ad2123b6611b6ceff018039df17f32ada8b534e6aa039545a3efb2"}, {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7d0c8399fcc1848491f00e0314bd59fb34a9c008761bcb422a057670c3f65e35"}, {file = "pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39"}, ] [[package]] name = "pygments" version = "2.18.0" requires_python = ">=3.8" summary = "Pygments is a syntax highlighting package written in Python." groups = ["doc"] files = [ {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, ] [[package]] name = "pytest" version = "8.3.4" requires_python = ">=3.8" summary = "pytest: simple powerful testing with Python" groups = ["lint", "test"] dependencies = [ "colorama; sys_platform == \"win32\"", "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", "iniconfig", "packaging", "pluggy<2,>=1.5", "tomli>=1; python_version < \"3.11\"", ] files = [ {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, ] [[package]] name = "pytest-cov" version = "5.0.0" requires_python = ">=3.8" summary = "Pytest plugin for measuring coverage." groups = ["test"] dependencies = [ "coverage[toml]>=5.2.1", "pytest>=4.6", ] files = [ {file = "pytest-cov-5.0.0.tar.gz", hash = "sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857"}, {file = "pytest_cov-5.0.0-py3-none-any.whl", hash = "sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652"}, ] [[package]] name = "pytest-lazy-fixtures" version = "1.1.1" requires_python = "<4.0,>=3.8" summary = "Allows you to use fixtures in @pytest.mark.parametrize." groups = ["test"] dependencies = [ "pytest>=7", ] files = [ {file = "pytest_lazy_fixtures-1.1.1-py3-none-any.whl", hash = "sha256:a4b396a361faf56c6305535fd0175ce82902ca7cf668c4d812a25ed2bcde8183"}, {file = "pytest_lazy_fixtures-1.1.1.tar.gz", hash = "sha256:0c561f0d29eea5b55cf29b9264a3241999ffdb74c6b6e8c4ccc0bd2c934d01ed"}, ] [[package]] name = "pytest-mock" version = "3.14.0" requires_python = ">=3.8" summary = "Thin-wrapper around the mock package for easier use with pytest" groups = ["test"] dependencies = [ "pytest>=6.2.5", ] files = [ {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, ] [[package]] name = "pytest-timeout" version = "2.3.1" requires_python = ">=3.7" summary = "pytest plugin to abort hanging tests" groups = ["test"] dependencies = [ "pytest>=7.0.0", ] files = [ {file = "pytest-timeout-2.3.1.tar.gz", hash = "sha256:12397729125c6ecbdaca01035b9e5239d4db97352320af155b3f5de1ba5165d9"}, {file = "pytest_timeout-2.3.1-py3-none-any.whl", hash = "sha256:68188cb703edfc6a18fad98dc25a3c61e9f24d644b0b70f33af545219fc7813e"}, ] [[package]] name = "pytest-xdist" version = "3.6.1" requires_python = ">=3.8" summary = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" groups = ["test"] dependencies = [ "execnet>=2.1", "pytest>=7.0.0", ] files = [ {file = "pytest_xdist-3.6.1-py3-none-any.whl", hash = "sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7"}, {file = "pytest_xdist-3.6.1.tar.gz", hash = "sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d"}, ] [[package]] name = "pytz" version = "2024.2" summary = "World timezone definitions, modern and historical" groups = ["doc"] marker = "python_version < \"3.9\"" files = [ {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, ] [[package]] name = "pywin32" version = "308" summary = "Python for Window Extensions" groups = ["test"] marker = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\"" files = [ {file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"}, {file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"}, {file = "pywin32-308-cp310-cp310-win_arm64.whl", hash = "sha256:a5ab5381813b40f264fa3495b98af850098f814a25a63589a8e9eb12560f450c"}, {file = "pywin32-308-cp311-cp311-win32.whl", hash = "sha256:5d8c8015b24a7d6855b1550d8e660d8daa09983c80e5daf89a273e5c6fb5095a"}, {file = "pywin32-308-cp311-cp311-win_amd64.whl", hash = "sha256:575621b90f0dc2695fec346b2d6302faebd4f0f45c05ea29404cefe35d89442b"}, {file = "pywin32-308-cp311-cp311-win_arm64.whl", hash = "sha256:100a5442b7332070983c4cd03f2e906a5648a5104b8a7f50175f7906efd16bb6"}, {file = "pywin32-308-cp312-cp312-win32.whl", hash = "sha256:587f3e19696f4bf96fde9d8a57cec74a57021ad5f204c9e627e15c33ff568897"}, {file = "pywin32-308-cp312-cp312-win_amd64.whl", hash = "sha256:00b3e11ef09ede56c6a43c71f2d31857cf7c54b0ab6e78ac659497abd2834f47"}, {file = "pywin32-308-cp312-cp312-win_arm64.whl", hash = "sha256:9b4de86c8d909aed15b7011182c8cab38c8850de36e6afb1f0db22b8959e3091"}, {file = "pywin32-308-cp313-cp313-win32.whl", hash = "sha256:1c44539a37a5b7b21d02ab34e6a4d314e0788f1690d65b48e9b0b89f31abbbed"}, {file = "pywin32-308-cp313-cp313-win_amd64.whl", hash = "sha256:fd380990e792eaf6827fcb7e187b2b4b1cede0585e3d0c9e84201ec27b9905e4"}, {file = "pywin32-308-cp313-cp313-win_arm64.whl", hash = "sha256:ef313c46d4c18dfb82a2431e3051ac8f112ccee1a34f29c263c583c568db63cd"}, {file = "pywin32-308-cp38-cp38-win32.whl", hash = "sha256:5794e764ebcabf4ff08c555b31bd348c9025929371763b2183172ff4708152f0"}, {file = "pywin32-308-cp38-cp38-win_amd64.whl", hash = "sha256:3b92622e29d651c6b783e368ba7d6722b1634b8e70bd376fd7610fe1992e19de"}, {file = "pywin32-308-cp39-cp39-win32.whl", hash = "sha256:7873ca4dc60ab3287919881a7d4f88baee4a6e639aa6962de25a98ba6b193341"}, {file = "pywin32-308-cp39-cp39-win_amd64.whl", hash = "sha256:71b3322d949b4cc20776436a9c9ba0eeedcbc9c650daa536df63f0ff111bb920"}, ] [[package]] name = "requests" version = "2.32.3" requires_python = ">=3.8" summary = "Python HTTP for Humans." groups = ["default", "doc"] dependencies = [ "certifi>=2017.4.17", "charset-normalizer<4,>=2", "idna<4,>=2.5", "urllib3<3,>=1.21.1", ] files = [ {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [[package]] name = "ruff" version = "0.8.4" requires_python = ">=3.7" summary = "An extremely fast Python linter and code formatter, written in Rust." groups = ["format", "lint"] files = [ {file = "ruff-0.8.4-py3-none-linux_armv6l.whl", hash = "sha256:58072f0c06080276804c6a4e21a9045a706584a958e644353603d36ca1eb8a60"}, {file = "ruff-0.8.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ffb60904651c00a1e0b8df594591770018a0f04587f7deeb3838344fe3adabac"}, {file = "ruff-0.8.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6ddf5d654ac0d44389f6bf05cee4caeefc3132a64b58ea46738111d687352296"}, {file = "ruff-0.8.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e248b1f0fa2749edd3350a2a342b67b43a2627434c059a063418e3d375cfe643"}, {file = "ruff-0.8.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf197b98ed86e417412ee3b6c893f44c8864f816451441483253d5ff22c0e81e"}, {file = "ruff-0.8.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c41319b85faa3aadd4d30cb1cffdd9ac6b89704ff79f7664b853785b48eccdf3"}, {file = "ruff-0.8.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:9f8402b7c4f96463f135e936d9ab77b65711fcd5d72e5d67597b543bbb43cf3f"}, {file = "ruff-0.8.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e4e56b3baa9c23d324ead112a4fdf20db9a3f8f29eeabff1355114dd96014604"}, {file = "ruff-0.8.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:736272574e97157f7edbbb43b1d046125fce9e7d8d583d5d65d0c9bf2c15addf"}, {file = "ruff-0.8.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5fe710ab6061592521f902fca7ebcb9fabd27bc7c57c764298b1c1f15fff720"}, {file = "ruff-0.8.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:13e9ec6d6b55f6da412d59953d65d66e760d583dd3c1c72bf1f26435b5bfdbae"}, {file = "ruff-0.8.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:97d9aefef725348ad77d6db98b726cfdb075a40b936c7984088804dfd38268a7"}, {file = "ruff-0.8.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ab78e33325a6f5374e04c2ab924a3367d69a0da36f8c9cb6b894a62017506111"}, {file = "ruff-0.8.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8ef06f66f4a05c3ddbc9121a8b0cecccd92c5bf3dd43b5472ffe40b8ca10f0f8"}, {file = "ruff-0.8.4-py3-none-win32.whl", hash = "sha256:552fb6d861320958ca5e15f28b20a3d071aa83b93caee33a87b471f99a6c0835"}, {file = "ruff-0.8.4-py3-none-win_amd64.whl", hash = "sha256:f21a1143776f8656d7f364bd264a9d60f01b7f52243fbe90e7670c0dfe0cf65d"}, {file = "ruff-0.8.4-py3-none-win_arm64.whl", hash = "sha256:9183dd615d8df50defa8b1d9a074053891ba39025cf5ae88e8bcb52edcc4bf08"}, {file = "ruff-0.8.4.tar.gz", hash = "sha256:0d5f89f254836799af1615798caa5f80b7f935d7a670fad66c5007928e57ace8"}, ] [[package]] name = "sadisplay" version = "0.4.9" summary = "SqlAlchemy schema display script" groups = ["doc"] dependencies = [ "SQLAlchemy>=0.5", ] files = [ {file = "sadisplay-0.4.9-py2.py3-none-any.whl", hash = "sha256:bf456f582b8f5da19fedef7a9afe969b49231d79724710bc7d35c9439f44c2fc"}, {file = "sadisplay-0.4.9.tar.gz", hash = "sha256:af67160f89123886ab42b247262862bfcde0a3c236229ecdd59de0a1e8e35d96"}, ] [[package]] name = "semantic-version" version = "2.10.0" requires_python = ">=2.7" summary = "A library implementing the 'SemVer' scheme." groups = ["lint"] files = [ {file = "semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177"}, {file = "semantic_version-2.10.0.tar.gz", hash = "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c"}, ] [[package]] name = "setuptools" version = "75.3.0" requires_python = ">=3.8" summary = "Easily download, build, install, upgrade, and uninstall Python packages" groups = ["lint", "release"] files = [ {file = "setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd"}, {file = "setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686"}, ] [[package]] name = "snowballstemmer" version = "2.2.0" summary = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." groups = ["doc"] files = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, ] [[package]] name = "sphinx" version = "5.3.0" requires_python = ">=3.6" summary = "Python documentation generator" groups = ["doc"] dependencies = [ "Jinja2>=3.0", "Pygments>=2.12", "alabaster<0.8,>=0.7", "babel>=2.9", "colorama>=0.4.5; sys_platform == \"win32\"", "docutils<0.20,>=0.14", "imagesize>=1.3", "importlib-metadata>=4.8; python_version < \"3.10\"", "packaging>=21.0", "requests>=2.5.0", "snowballstemmer>=2.0", "sphinxcontrib-applehelp", "sphinxcontrib-devhelp", "sphinxcontrib-htmlhelp>=2.0.0", "sphinxcontrib-jsmath", "sphinxcontrib-qthelp", "sphinxcontrib-serializinghtml>=1.1.5", ] files = [ {file = "Sphinx-5.3.0.tar.gz", hash = "sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5"}, {file = "sphinx-5.3.0-py3-none-any.whl", hash = "sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d"}, ] [[package]] name = "sphinx-autobuild" version = "2021.3.14" requires_python = ">=3.6" summary = "Rebuild Sphinx documentation on changes, with live-reload in the browser." groups = ["doc"] dependencies = [ "colorama", "livereload", "sphinx", ] files = [ {file = "sphinx-autobuild-2021.3.14.tar.gz", hash = "sha256:de1ca3b66e271d2b5b5140c35034c89e47f263f2cd5db302c9217065f7443f05"}, {file = "sphinx_autobuild-2021.3.14-py3-none-any.whl", hash = "sha256:8fe8cbfdb75db04475232f05187c776f46f6e9e04cacf1e49ce81bdac649ccac"}, ] [[package]] name = "sphinx-autodoc-typehints" version = "1.23.0" requires_python = ">=3.7" summary = "Type hints (PEP 484) support for the Sphinx autodoc extension" groups = ["doc"] dependencies = [ "sphinx>=5.3", ] files = [ {file = "sphinx_autodoc_typehints-1.23.0-py3-none-any.whl", hash = "sha256:ac099057e66b09e51b698058ba7dd76e57e1fe696cd91b54e121d3dad188f91d"}, {file = "sphinx_autodoc_typehints-1.23.0.tar.gz", hash = "sha256:5d44e2996633cdada499b6d27a496ddf9dbc95dd1f0f09f7b37940249e61f6e9"}, ] [[package]] name = "sphinx-rtd-theme" version = "2.0.0" requires_python = ">=3.6" summary = "Read the Docs theme for Sphinx" groups = ["doc"] dependencies = [ "docutils<0.21", "sphinx<8,>=5", "sphinxcontrib-jquery<5,>=4", ] files = [ {file = "sphinx_rtd_theme-2.0.0-py2.py3-none-any.whl", hash = "sha256:ec93d0856dc280cf3aee9a4c9807c60e027c7f7b461b77aeffed682e68f0e586"}, {file = "sphinx_rtd_theme-2.0.0.tar.gz", hash = "sha256:bd5d7b80622406762073a04ef8fadc5f9151261563d47027de09910ce03afe6b"}, ] [[package]] name = "sphinxcontrib-applehelp" version = "1.0.4" requires_python = ">=3.8" summary = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" groups = ["doc"] files = [ {file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"}, {file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"}, ] [[package]] name = "sphinxcontrib-devhelp" version = "1.0.2" requires_python = ">=3.5" summary = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." groups = ["doc"] files = [ {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"}, {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"}, ] [[package]] name = "sphinxcontrib-htmlhelp" version = "2.0.1" requires_python = ">=3.8" summary = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" groups = ["doc"] files = [ {file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"}, {file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"}, ] [[package]] name = "sphinxcontrib-jquery" version = "4.1" requires_python = ">=2.7" summary = "Extension to include jQuery on newer Sphinx releases" groups = ["doc"] dependencies = [ "Sphinx>=1.8", ] files = [ {file = "sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a"}, {file = "sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae"}, ] [[package]] name = "sphinxcontrib-jsmath" version = "1.0.1" requires_python = ">=3.5" summary = "A sphinx extension which renders display math in HTML via JavaScript" groups = ["doc"] files = [ {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, ] [[package]] name = "sphinxcontrib-plantuml" version = "0.30" summary = "Sphinx \"plantuml\" extension" groups = ["doc"] dependencies = [ "Sphinx>=1.6", ] files = [ {file = "sphinxcontrib-plantuml-0.30.tar.gz", hash = "sha256:2a1266ca43bddf44640ae44107003df4490de2b3c3154a0d627cfb63e9a169bf"}, ] [[package]] name = "sphinxcontrib-qthelp" version = "1.0.3" requires_python = ">=3.5" summary = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." groups = ["doc"] files = [ {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"}, {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"}, ] [[package]] name = "sphinxcontrib-serializinghtml" version = "1.1.5" requires_python = ">=3.5" summary = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." groups = ["doc"] files = [ {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"}, {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"}, ] [[package]] name = "sqlalchemy" version = "2.0.36" requires_python = ">=3.7" summary = "Database Abstraction Library" groups = ["doc"] dependencies = [ "greenlet!=0.4.17; (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\") and python_version < \"3.13\"", "importlib-metadata; python_version < \"3.8\"", "typing-extensions>=4.6.0", ] files = [ {file = "SQLAlchemy-2.0.36-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59b8f3adb3971929a3e660337f5dacc5942c2cdb760afcabb2614ffbda9f9f72"}, {file = "SQLAlchemy-2.0.36-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37350015056a553e442ff672c2d20e6f4b6d0b2495691fa239d8aa18bb3bc908"}, {file = "SQLAlchemy-2.0.36-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8318f4776c85abc3f40ab185e388bee7a6ea99e7fa3a30686580b209eaa35c08"}, {file = "SQLAlchemy-2.0.36-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c245b1fbade9c35e5bd3b64270ab49ce990369018289ecfde3f9c318411aaa07"}, {file = "SQLAlchemy-2.0.36-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:69f93723edbca7342624d09f6704e7126b152eaed3cdbb634cb657a54332a3c5"}, {file = "SQLAlchemy-2.0.36-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f9511d8dd4a6e9271d07d150fb2f81874a3c8c95e11ff9af3a2dfc35fe42ee44"}, {file = "SQLAlchemy-2.0.36-cp310-cp310-win32.whl", hash = "sha256:c3f3631693003d8e585d4200730616b78fafd5a01ef8b698f6967da5c605b3fa"}, {file = "SQLAlchemy-2.0.36-cp310-cp310-win_amd64.whl", hash = "sha256:a86bfab2ef46d63300c0f06936bd6e6c0105faa11d509083ba8f2f9d237fb5b5"}, {file = "SQLAlchemy-2.0.36-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fd3a55deef00f689ce931d4d1b23fa9f04c880a48ee97af488fd215cf24e2a6c"}, {file = "SQLAlchemy-2.0.36-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f5e9cd989b45b73bd359f693b935364f7e1f79486e29015813c338450aa5a71"}, {file = "SQLAlchemy-2.0.36-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ddd9db6e59c44875211bc4c7953a9f6638b937b0a88ae6d09eb46cced54eff"}, {file = "SQLAlchemy-2.0.36-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2519f3a5d0517fc159afab1015e54bb81b4406c278749779be57a569d8d1bb0d"}, {file = "SQLAlchemy-2.0.36-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:59b1ee96617135f6e1d6f275bbe988f419c5178016f3d41d3c0abb0c819f75bb"}, {file = "SQLAlchemy-2.0.36-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:39769a115f730d683b0eb7b694db9789267bcd027326cccc3125e862eb03bfd8"}, {file = "SQLAlchemy-2.0.36-cp311-cp311-win32.whl", hash = "sha256:66bffbad8d6271bb1cc2f9a4ea4f86f80fe5e2e3e501a5ae2a3dc6a76e604e6f"}, {file = "SQLAlchemy-2.0.36-cp311-cp311-win_amd64.whl", hash = "sha256:23623166bfefe1487d81b698c423f8678e80df8b54614c2bf4b4cfcd7c711959"}, {file = "SQLAlchemy-2.0.36-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7b64e6ec3f02c35647be6b4851008b26cff592a95ecb13b6788a54ef80bbdd4"}, {file = "SQLAlchemy-2.0.36-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:46331b00096a6db1fdc052d55b101dbbfc99155a548e20a0e4a8e5e4d1362855"}, {file = "SQLAlchemy-2.0.36-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdf3386a801ea5aba17c6410dd1dc8d39cf454ca2565541b5ac42a84e1e28f53"}, {file = "SQLAlchemy-2.0.36-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9dfa18ff2a67b09b372d5db8743c27966abf0e5344c555d86cc7199f7ad83a"}, {file = "SQLAlchemy-2.0.36-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:90812a8933df713fdf748b355527e3af257a11e415b613dd794512461eb8a686"}, {file = "SQLAlchemy-2.0.36-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1bc330d9d29c7f06f003ab10e1eaced295e87940405afe1b110f2eb93a233588"}, {file = "SQLAlchemy-2.0.36-cp312-cp312-win32.whl", hash = "sha256:79d2e78abc26d871875b419e1fd3c0bca31a1cb0043277d0d850014599626c2e"}, {file = "SQLAlchemy-2.0.36-cp312-cp312-win_amd64.whl", hash = "sha256:b544ad1935a8541d177cb402948b94e871067656b3a0b9e91dbec136b06a2ff5"}, {file = "SQLAlchemy-2.0.36-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b5cc79df7f4bc3d11e4b542596c03826063092611e481fcf1c9dfee3c94355ef"}, {file = "SQLAlchemy-2.0.36-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3c01117dd36800f2ecaa238c65365b7b16497adc1522bf84906e5710ee9ba0e8"}, {file = "SQLAlchemy-2.0.36-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bc633f4ee4b4c46e7adcb3a9b5ec083bf1d9a97c1d3854b92749d935de40b9b"}, {file = "SQLAlchemy-2.0.36-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e46ed38affdfc95d2c958de328d037d87801cfcbea6d421000859e9789e61c2"}, {file = "SQLAlchemy-2.0.36-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b2985c0b06e989c043f1dc09d4fe89e1616aadd35392aea2844f0458a989eacf"}, {file = "SQLAlchemy-2.0.36-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a121d62ebe7d26fec9155f83f8be5189ef1405f5973ea4874a26fab9f1e262c"}, {file = "SQLAlchemy-2.0.36-cp313-cp313-win32.whl", hash = "sha256:0572f4bd6f94752167adfd7c1bed84f4b240ee6203a95e05d1e208d488d0d436"}, {file = "SQLAlchemy-2.0.36-cp313-cp313-win_amd64.whl", hash = "sha256:8c78ac40bde930c60e0f78b3cd184c580f89456dd87fc08f9e3ee3ce8765ce88"}, {file = "SQLAlchemy-2.0.36-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3d6718667da04294d7df1670d70eeddd414f313738d20a6f1d1f379e3139a545"}, {file = "SQLAlchemy-2.0.36-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:72c28b84b174ce8af8504ca28ae9347d317f9dba3999e5981a3cd441f3712e24"}, {file = "SQLAlchemy-2.0.36-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b11d0cfdd2b095e7b0686cf5fabeb9c67fae5b06d265d8180715b8cfa86522e3"}, {file = "SQLAlchemy-2.0.36-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e32092c47011d113dc01ab3e1d3ce9f006a47223b18422c5c0d150af13a00687"}, {file = "SQLAlchemy-2.0.36-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:6a440293d802d3011028e14e4226da1434b373cbaf4a4bbb63f845761a708346"}, {file = "SQLAlchemy-2.0.36-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c54a1e53a0c308a8e8a7dffb59097bff7facda27c70c286f005327f21b2bd6b1"}, {file = "SQLAlchemy-2.0.36-cp38-cp38-win32.whl", hash = "sha256:1e0d612a17581b6616ff03c8e3d5eff7452f34655c901f75d62bd86449d9750e"}, {file = "SQLAlchemy-2.0.36-cp38-cp38-win_amd64.whl", hash = "sha256:8958b10490125124463095bbdadda5aa22ec799f91958e410438ad6c97a7b793"}, {file = "SQLAlchemy-2.0.36-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc022184d3e5cacc9579e41805a681187650e170eb2fd70e28b86192a479dcaa"}, {file = "SQLAlchemy-2.0.36-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b817d41d692bf286abc181f8af476c4fbef3fd05e798777492618378448ee689"}, {file = "SQLAlchemy-2.0.36-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4e46a888b54be23d03a89be510f24a7652fe6ff660787b96cd0e57a4ebcb46d"}, {file = "SQLAlchemy-2.0.36-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4ae3005ed83f5967f961fd091f2f8c5329161f69ce8480aa8168b2d7fe37f06"}, {file = "SQLAlchemy-2.0.36-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:03e08af7a5f9386a43919eda9de33ffda16b44eb11f3b313e6822243770e9763"}, {file = "SQLAlchemy-2.0.36-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3dbb986bad3ed5ceaf090200eba750b5245150bd97d3e67343a3cfed06feecf7"}, {file = "SQLAlchemy-2.0.36-cp39-cp39-win32.whl", hash = "sha256:9fe53b404f24789b5ea9003fc25b9a3988feddebd7e7b369c8fac27ad6f52f28"}, {file = "SQLAlchemy-2.0.36-cp39-cp39-win_amd64.whl", hash = "sha256:af148a33ff0349f53512a049c6406923e4e02bf2f26c5fb285f143faf4f0e46a"}, {file = "SQLAlchemy-2.0.36-py3-none-any.whl", hash = "sha256:fddbe92b4760c6f5d48162aef14824add991aeda8ddadb3c31d56eb15ca69f8e"}, {file = "sqlalchemy-2.0.36.tar.gz", hash = "sha256:7f2767680b6d2398aea7082e45a774b2b0767b5c8d8ffb9c8b683088ea9b29c5"}, ] [[package]] name = "toml" version = "0.10.2" requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" summary = "Python Library for Tom's Obvious, Minimal Language" groups = ["lint"] files = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] [[package]] name = "tomli" version = "2.2.1" requires_python = ">=3.8" summary = "A lil' TOML parser" groups = ["lint", "release", "test"] marker = "python_version < \"3.11\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] [[package]] name = "tornado" version = "6.4.2" requires_python = ">=3.8" summary = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." groups = ["doc"] files = [ {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, ] [[package]] name = "towncrier" version = "23.11.0" requires_python = ">=3.8" summary = "Building newsfiles for your project." groups = ["release"] marker = "python_version >= \"3.8\"" dependencies = [ "click", "importlib-resources>=5; python_version < \"3.10\"", "incremental", "jinja2", "tomli; python_version < \"3.11\"", ] files = [ {file = "towncrier-23.11.0-py3-none-any.whl", hash = "sha256:2e519ca619426d189e3c98c99558fe8be50c9ced13ea1fc20a4a353a95d2ded7"}, {file = "towncrier-23.11.0.tar.gz", hash = "sha256:13937c247e3f8ae20ac44d895cf5f96a60ad46cfdcc1671759530d7837d9ee5d"}, ] [[package]] name = "tqdm" version = "4.67.1" requires_python = ">=3.7" summary = "Fast, Extensible Progress Meter" groups = ["test"] dependencies = [ "colorama; platform_system == \"Windows\"", ] files = [ {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, ] [[package]] name = "typing-extensions" version = "4.12.2" requires_python = ">=3.8" summary = "Backported and Experimental Type Hints for Python 3.8+" groups = ["default", "doc", "test"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [[package]] name = "urllib3" version = "2.2.3" requires_python = ">=3.8" summary = "HTTP library with thread-safe connection pooling, file post, and more." groups = ["default", "doc"] files = [ {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, ] [[package]] name = "zipp" version = "3.20.2" requires_python = ">=3.8" summary = "Backport of pathlib-compatible object wrapper for zip files" groups = ["doc", "release"] marker = "python_version < \"3.10\"" files = [ {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, ] b2-sdk-python-2.8.0/pyproject.toml000066400000000000000000000107501474454370000170510ustar00rootroot00000000000000[project] name = "b2sdk" description = "Backblaze B2 SDK" dynamic = ["version"] authors = [ {name = "Backblaze Inc", email = "support@backblaze.com"}, ] dependencies = [ "annotated_types>=0.5.0", "importlib-metadata>=3.3.0; python_version < '3.8'", "logfury<2.0.0,>=1.0.1", "requests<3.0.0,>=2.9.1", "typing-extensions>=4.7.1; python_version < '3.12'", ] requires-python = ">=3.8" readme = "README.md" license = {text = "MIT"} keywords = ["backblaze", "b2", "cloud", "storage"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Topic :: Software Development :: Libraries", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] [project.urls] Homepage = "https://github.com/Backblaze/b2-sdk-python" [project.entry-points.pyinstaller40] hook-dirs = "b2sdk._pyinstaller:get_hook_dirs" [build-system] requires = ["pdm-backend"] build-backend = "pdm.backend" [tool.liccheck] authorized_licenses = [ "bsd", "new bsd", "bsd license", "new bsd license", "simplified bsd", "apache", "apache 2.0", "apache software", "apache software license", "lgpl", "gnu lgpl", "gnu library or lesser general public license (lgpl)", "lgpl with exceptions or zpl", "isc license", "isc license (iscl)", "mit", "mit license", "mozilla public license 2.0 (mpl 2.0)", "mpl-2.0", "psf", "python software foundation", "python software foundation license", "zpl 2.1", ] unauthorized_licences = [ "affero", "agpl", "gpl v3", "gpl v2", "gpl", ] dependencies = true optional_dependencies = ["full", "doc"] [tool.ruff] # TODO add D select = ["E", "F", "I", "UP"] # TODO: remove E501 once docstrings are formatted ignore = [ "D100", "D105", "D107", "D200", "D202", "D203", "D205", "D212", "D400", "D401", "D415", "D101", "D102","D103", "D104", # TODO remove once we have docstring for all public methods "E501", # TODO: remove E501 once docstrings are formatted "UP031", ] line-length = 100 target-version = "py37" [tool.ruff.format] quote-style = "single" [tool.ruff.per-file-ignores] "__init__.py" = ["I", "F401"] "b2sdk/_v3/__init__.py" = ["E402"] "b2sdk/v*/**" = ["I", "F403", "F405"] "b2sdk/_v*/**" = ["I", "F403", "F405"] "test/**" = ["D", "F403", "F405"] [tool.towncrier] directory = "changelog.d" filename = "CHANGELOG.md" start_string = "\n" underlines = ["", "", ""] title_format = "## [{version}](https://github.com/Backblaze/b2-sdk-python/releases/tag/v{version}) - {project_date}" issue_format = "[#{issue}](https://github.com/Backblaze/b2-sdk-python/issues/{issue})" [[tool.towncrier.type]] directory = "removed" name = "Removed" showcontent = true [[tool.towncrier.type]] directory = "changed" name = "Changed" showcontent = true [[tool.towncrier.type]] directory = "fixed" name = "Fixed" showcontent = true [[tool.towncrier.type]] directory = "deprecated" name = "Deprecated" showcontent = true [[tool.towncrier.type]] directory = "added" name = "Added" showcontent = true [[tool.towncrier.type]] directory = "doc" name = "Doc" showcontent = true [[tool.towncrier.type]] directory = "infrastructure" name = "Infrastructure" showcontent = true [tool.pdm] distribution = true [tool.pdm.build] includes = ["b2sdk"] [tool.pdm.version] source = "scm" [tool.pdm.dev-dependencies] format = [ "ruff~=0.8.4", ] lint = [ "ruff~=0.8.4", "pytest>=8.3.3", "liccheck==0.9.2", "setuptools", ] test = [ "coverage>=7.2.7", "pytest>=8.3.3", "pytest-cov>=3.0.0", "pytest-mock>=3.6.1", "pytest-lazy-fixtures==1.1.1", "pytest-xdist>=2.5.0", "pytest-timeout>=2.1.0", "tqdm<5.0.0,>=4.5.0", "eval_type_backport>=0.1.3,<1; python_version<'3.10'", # used with pydantic "pydantic>=2.0.1", "pywin32>=306; sys_platform == \"win32\" and platform_python_implementation!='PyPy'", ] release = [ "towncrier==23.11.0; python_version>='3.8'", ] doc = [ "sadisplay>=0.4.9", "sphinx>=5.3.0, <6", "sphinx-autobuild>=2021.3.14", "sphinx-rtd-theme>=2.0.0", "sphinx-autodoc-typehints>=1.23.0", "sphinxcontrib-plantuml>=0.27", "tornado>=6.3.3; python_version>='3.8'", ] b2-sdk-python-2.8.0/setup.cfg000066400000000000000000000000331474454370000157470ustar00rootroot00000000000000[coverage:run] branch=true b2-sdk-python-2.8.0/test/000077500000000000000000000000001474454370000151115ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/__init__.py000066400000000000000000000004761474454370000172310ustar00rootroot00000000000000###################################################################### # # File: test/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/conftest.py000066400000000000000000000012611474454370000173100ustar00rootroot00000000000000###################################################################### # # File: test/conftest.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import concurrent.futures import pytest @pytest.fixture def bg_executor(): with concurrent.futures.ThreadPoolExecutor() as executor: yield executor @pytest.fixture def apiver_module(): """ b2sdk apiver module fixture. A compatibility function that is to be replaced by `pytest-apiver` plugin in the future. """ import apiver_deps # noqa: F401 return apiver_deps b2-sdk-python-2.8.0/test/helpers.py000066400000000000000000000046771474454370000171430ustar00rootroot00000000000000###################################################################### # # File: test/helpers.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import contextlib import inspect import io from unittest.mock import patch from b2sdk._internal.types import pydantic @contextlib.contextmanager def patch_bind_params(instance, method_name): """ Patch a method of instance. In addition to `patch.object(instance, method_name)` would provide, it also adds get_bound_call_args method on the returned mock. This allows to get the arguments that were passed to the method, after binding. :param instance: instance to patch :param method_name: name of the method of instance to patch :return: patched method mock """ signature = inspect.signature(getattr(instance, method_name)) with patch.object(instance, method_name, autospec=True) as mock_method: mock_method.get_bound_call_args = lambda: signature.bind( *mock_method.call_args[0], **mock_method.call_args[1] ).arguments yield mock_method class NonSeekableIO(io.BytesIO): """Emulate a non-seekable file""" def seek(self, *args, **kwargs): raise OSError('not seekable') def seekable(self): return False def type_validator_factory(type_): """ Equivalent of `TypeAdapter(type_).validate_python` and noop under Python <3.8. To be removed when we drop support for Python <3.8. """ if pydantic: return pydantic.TypeAdapter(type_).validate_python return lambda *args, **kwargs: None def deep_cast_dict(actual, expected): """ For composite objects `actual` and `expected`, return a copy of `actual` (with all dicts and lists deeply copied) with all keys of dicts not appearing in `expected` (comparing dicts on any level) removed. Useful for assertions in tests ignoring extra keys. """ if isinstance(expected, dict) and isinstance(actual, dict): return {k: deep_cast_dict(actual[k], expected[k]) for k in expected if k in actual} elif isinstance(expected, list) and isinstance(actual, list): return [deep_cast_dict(a, e) for a, e in zip(actual, expected)] return actual def assert_dict_equal_ignore_extra(actual, expected): assert deep_cast_dict(actual, expected) == expected b2-sdk-python-2.8.0/test/integration/000077500000000000000000000000001474454370000174345ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/integration/__init__.py000066400000000000000000000013521474454370000215460ustar00rootroot00000000000000###################################################################### # # File: test/integration/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import os def get_b2_auth_data(): application_key_id = os.environ.get('B2_TEST_APPLICATION_KEY_ID') if application_key_id is None: raise ValueError('B2_TEST_APPLICATION_KEY_ID is not set.') application_key = os.environ.get('B2_TEST_APPLICATION_KEY') if application_key is None: raise ValueError('B2_TEST_APPLICATION_KEY is not set.') return application_key_id, application_key b2-sdk-python-2.8.0/test/integration/base.py000066400000000000000000000057721474454370000207330ustar00rootroot00000000000000###################################################################### # # File: test/integration/base.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from b2sdk.v2 import B2Api, current_time_millis from b2sdk.v2.exception import DuplicateBucketName from test.integration.bucket_cleaner import BucketCleaner from test.integration.helpers import ( BUCKET_CREATED_AT_MILLIS, random_bucket_name, ) @pytest.mark.usefixtures('cls_setup') class IntegrationTestBase: b2_api: B2Api this_run_bucket_name_prefix: str bucket_cleaner: BucketCleaner @pytest.fixture(autouse=True, scope='class') def cls_setup(self, request, b2_api, b2_auth_data, bucket_name_prefix, bucket_cleaner): cls = request.cls cls.b2_auth_data = b2_auth_data cls.this_run_bucket_name_prefix = bucket_name_prefix cls.bucket_cleaner = bucket_cleaner cls.b2_api = b2_api cls.info = b2_api.account_info @pytest.fixture(autouse=True) def setup_method(self): self.buckets_created = [] yield for bucket in self.buckets_created: self.bucket_cleaner.cleanup_bucket(bucket) def generate_bucket_name(self): return random_bucket_name(self.this_run_bucket_name_prefix) def write_zeros(self, file, number): line = b'0' * 1000 + b'\n' line_len = len(line) written = 0 while written <= number: file.write(line) written += line_len def create_bucket(self): bucket_name = self.generate_bucket_name() try: bucket = self.b2_api.create_bucket( bucket_name, 'allPublic', bucket_info={BUCKET_CREATED_AT_MILLIS: str(current_time_millis())}, ) except DuplicateBucketName: self._duplicated_bucket_name_debug_info(bucket_name) raise self.buckets_created.append(bucket) return bucket def _duplicated_bucket_name_debug_info(self, bucket_name: str) -> None: # Trying to obtain as much information as possible about this bucket. print(' DUPLICATED BUCKET DEBUG START '.center(60, '=')) bucket = self.b2_api.get_bucket_by_name(bucket_name) print('Bucket metadata:') bucket_dict = bucket.as_dict() for info_key, info in bucket_dict.items(): print(f'\t{info_key}: "{info}"') print('All files (and their versions) inside the bucket:') ls_generator = bucket.ls(recursive=True, latest_only=False) for file_version, _directory in ls_generator: # as_dict() is bound to have more info than we can use, # but maybe some of it will cast some light on the issue. print(f'\t{file_version.file_name} ({file_version.as_dict()})') print(' DUPLICATED BUCKET DEBUG END '.center(60, '=')) b2-sdk-python-2.8.0/test/integration/bucket_cleaner.py000066400000000000000000000113721474454370000227600ustar00rootroot00000000000000###################################################################### # # File: test/integration/bucket_cleaner.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging from b2sdk.v2 import ( NO_RETENTION_FILE_SETTING, B2Api, Bucket, LegalHold, RetentionMode, current_time_millis, ) from b2sdk.v2.exception import BadRequest from .helpers import BUCKET_CREATED_AT_MILLIS, GENERAL_BUCKET_NAME_PREFIX ONE_HOUR_MILLIS = 60 * 60 * 1000 logger = logging.getLogger(__name__) class BucketCleaner: def __init__( self, dont_cleanup_old_buckets: bool, b2_api: B2Api, current_run_prefix: str | None = None ): self.current_run_prefix = current_run_prefix self.dont_cleanup_old_buckets = dont_cleanup_old_buckets self.b2_api = b2_api def _should_remove_bucket(self, bucket: Bucket): if self.current_run_prefix and bucket.name.startswith(self.current_run_prefix): return True if self.dont_cleanup_old_buckets: return False if bucket.name.startswith(GENERAL_BUCKET_NAME_PREFIX): if BUCKET_CREATED_AT_MILLIS in bucket.bucket_info: if ( int(bucket.bucket_info[BUCKET_CREATED_AT_MILLIS]) < current_time_millis() - ONE_HOUR_MILLIS ): return True return False def cleanup_buckets(self): buckets = self.b2_api.list_buckets() for bucket in buckets: self.cleanup_bucket(bucket) def cleanup_bucket(self, bucket: Bucket): b2_api = self.b2_api if not self._should_remove_bucket(bucket): logger.info('Skipping bucket removal:', bucket.name) else: logger.info('Trying to remove bucket:', bucket.name) files_leftover = False try: b2_api.delete_bucket(bucket) except BadRequest: logger.info('Bucket is not empty, removing files') files_leftover = True if files_leftover: files_leftover = False file_versions = bucket.ls(latest_only=False, recursive=True) for file_version_info, _ in file_versions: if file_version_info.file_retention: if file_version_info.file_retention.mode == RetentionMode.GOVERNANCE: logger.info( 'Removing retention from file version: %s', file_version_info.id_ ) b2_api.update_file_retention( file_version_info.id_, file_version_info.file_name, NO_RETENTION_FILE_SETTING, True, ) elif file_version_info.file_retention.mode == RetentionMode.COMPLIANCE: if ( file_version_info.file_retention.retain_until > current_time_millis() ): logger.info( 'File version: %s cannot be removed due to compliance mode retention', file_version_info.id_, ) files_leftover = True continue elif file_version_info.file_retention.mode == RetentionMode.NONE: pass else: raise ValueError( f'Unknown retention mode: {file_version_info.file_retention.mode}' ) if file_version_info.legal_hold.is_on(): logger.info( 'Removing legal hold from file version: %s', file_version_info.id_ ) b2_api.update_file_legal_hold( file_version_info.id_, file_version_info.file_name, LegalHold.OFF ) logger.info('Removing file version:', file_version_info.id_) b2_api.delete_file_version(file_version_info.id_, file_version_info.file_name) if files_leftover: logger.info('Unable to remove bucket because some retained files remain') return else: b2_api.delete_bucket(bucket) logger.info('Removed bucket:', bucket.name) b2-sdk-python-2.8.0/test/integration/cleanup_buckets.py000077500000000000000000000012521474454370000231600ustar00rootroot00000000000000###################################################################### # # File: test/integration/cleanup_buckets.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from test.integration.helpers import authorize from . import get_b2_auth_data from .bucket_cleaner import BucketCleaner from .test_raw_api import cleanup_old_buckets if __name__ == '__main__': cleanup_old_buckets() BucketCleaner( dont_cleanup_old_buckets=False, b2_api=authorize(get_b2_auth_data())[0] ).cleanup_buckets() b2-sdk-python-2.8.0/test/integration/conftest.py000066400000000000000000000047441474454370000216440ustar00rootroot00000000000000###################################################################### # # File: test/integration/conftest.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import http import http.client import os import secrets import pytest from b2sdk._internal.utils import current_time_millis from test.integration import get_b2_auth_data from test.integration.bucket_cleaner import BucketCleaner from test.integration.helpers import ( BUCKET_CREATED_AT_MILLIS, authorize, get_bucket_name_prefix, random_bucket_name, ) def pytest_addoption(parser): """Add a flag for not cleaning up old buckets""" parser.addoption( '--dont-cleanup-old-buckets', action='store_true', default=False, ) @pytest.fixture(scope='session') def dont_cleanup_old_buckets(request): return request.config.getoption('--dont-cleanup-old-buckets') @pytest.fixture(autouse=True, scope='session') def set_http_debug(): if os.environ.get('B2_DEBUG_HTTP'): http.client.HTTPConnection.debuglevel = 1 @pytest.fixture(scope='session') def b2_auth_data(): try: return get_b2_auth_data() except ValueError as ex: pytest.fail(ex.args[0]) @pytest.fixture(scope='session') def bucket_name_prefix(): return get_bucket_name_prefix(8) @pytest.fixture(scope='session') def _b2_api(b2_auth_data): b2_api, _ = authorize(b2_auth_data) return b2_api @pytest.fixture(scope='session') def bucket_cleaner(bucket_name_prefix, dont_cleanup_old_buckets, _b2_api): cleaner = BucketCleaner( dont_cleanup_old_buckets, _b2_api, current_run_prefix=bucket_name_prefix, ) yield cleaner cleaner.cleanup_buckets() @pytest.fixture(scope='session') def b2_api(_b2_api, bucket_cleaner): return _b2_api @pytest.fixture def bucket(b2_api, bucket_name_prefix, bucket_cleaner): bucket = b2_api.create_bucket( random_bucket_name(bucket_name_prefix), 'allPrivate', bucket_info={ 'created_by': 'b2-sdk integration test', BUCKET_CREATED_AT_MILLIS: str(current_time_millis()), }, ) yield bucket bucket_cleaner.cleanup_bucket(bucket) @pytest.fixture def b2_subfolder(bucket, request): subfolder_name = f'{request.node.name}_{secrets.token_urlsafe(4)}' return f'b2://{bucket.name}/{subfolder_name}' b2-sdk-python-2.8.0/test/integration/helpers.py000066400000000000000000000025311474454370000214510ustar00rootroot00000000000000###################################################################### # # File: test/integration/helpers.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import os import secrets from b2sdk.v2 import ( BUCKET_NAME_CHARS_UNIQ, BUCKET_NAME_LENGTH_RANGE, DEFAULT_HTTP_API_CONFIG, B2Api, InMemoryAccountInfo, ) GENERAL_BUCKET_NAME_PREFIX = 'sdktst' BUCKET_NAME_LENGTH = BUCKET_NAME_LENGTH_RANGE[1] BUCKET_CREATED_AT_MILLIS = 'created_at_millis' RNG = secrets.SystemRandom() def _bucket_name_prefix_part(length: int) -> str: return ''.join(RNG.choice(BUCKET_NAME_CHARS_UNIQ) for _ in range(length)) def get_bucket_name_prefix(rnd_len: int = 8) -> str: return GENERAL_BUCKET_NAME_PREFIX + _bucket_name_prefix_part(rnd_len) def random_bucket_name(prefix: str = GENERAL_BUCKET_NAME_PREFIX) -> str: return prefix + _bucket_name_prefix_part(BUCKET_NAME_LENGTH - len(prefix)) def authorize(b2_auth_data, api_config=DEFAULT_HTTP_API_CONFIG): info = InMemoryAccountInfo() b2_api = B2Api(info, api_config=api_config) realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') b2_api.authorize_account(realm, *b2_auth_data) return b2_api, info b2-sdk-python-2.8.0/test/integration/test_bucket.py000066400000000000000000000040061474454370000223220ustar00rootroot00000000000000###################################################################### # # File: test/integration/test_bucket.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from test.helpers import assert_dict_equal_ignore_extra def test_bucket_notification_rules(bucket, b2_api): if 'writeBucketNotifications' not in b2_api.account_info.get_allowed()['capabilities']: pytest.skip('Test account does not have writeBucketNotifications capability') assert bucket.set_notification_rules([]) == [] assert bucket.get_notification_rules() == [] notification_rule = { 'eventTypes': ['b2:ObjectCreated:*'], 'isEnabled': True, 'name': 'test-rule', 'objectNamePrefix': '', 'targetConfiguration': { 'customHeaders': [], 'targetType': 'webhook', 'url': 'https://example.com/webhook', 'hmacSha256SigningSecret': 'stringOf32AlphaNumericCharacters', }, } set_notification_rules = bucket.set_notification_rules([notification_rule]) assert set_notification_rules == bucket.get_notification_rules() assert_dict_equal_ignore_extra( set_notification_rules, [{**notification_rule, 'isSuspended': False, 'suspensionReason': ''}], ) assert bucket.set_notification_rules([]) == [] def test_bucket_update__lifecycle_rules(bucket, b2_api): lifecycle_rule = { 'daysFromHidingToDeleting': 1, 'daysFromUploadingToHiding': 1, 'daysFromStartingToCancelingUnfinishedLargeFiles': 1, 'fileNamePrefix': '', } old_rules_list = bucket.lifecycle_rules updated_bucket = bucket.update(lifecycle_rules=[lifecycle_rule]) assert updated_bucket.lifecycle_rules == [lifecycle_rule] assert bucket.lifecycle_rules is old_rules_list updated_bucket = bucket.update(lifecycle_rules=[]) assert updated_bucket.lifecycle_rules == [] b2-sdk-python-2.8.0/test/integration/test_download.py000066400000000000000000000151271474454370000226620ustar00rootroot00000000000000###################################################################### # # File: test/integration/test_download.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import gzip import io import os import pathlib import platform import tempfile from pprint import pprint from unittest import mock import pytest from b2sdk._internal.utils import Sha1HexDigest from b2sdk._internal.utils.filesystem import _IS_WINDOWS from b2sdk.v2 import * from .base import IntegrationTestBase from .helpers import authorize class TestDownload(IntegrationTestBase): def test_large_file(self): bucket = self.create_bucket() with mock.patch.object( self.info, '_recommended_part_size', new=self.info.get_absolute_minimum_part_size() ): download_manager = self.b2_api.services.download_manager with mock.patch.object( download_manager, 'strategies', new=[ ParallelDownloader( min_part_size=self.info.get_absolute_minimum_part_size(), min_chunk_size=download_manager.MIN_CHUNK_SIZE, max_chunk_size=download_manager.MAX_CHUNK_SIZE, thread_pool=download_manager._thread_pool, ) ], ): # let's check that small file downloads do not fail with these settings small_file_version = bucket.upload_bytes(b'0', 'a_single_char') with io.BytesIO() as io_: bucket.download_file_by_name('a_single_char').save(io_) assert io_.getvalue() == b'0' f, sha1 = self._file_helper(bucket) if small_file_version._type() != 'large': # if we are here, that's not the production server! assert ( f.download_version.content_sha1_verified ) # large files don't have sha1, lets not check file_info = f.download_version.file_info assert LARGE_FILE_SHA1 in file_info assert file_info[LARGE_FILE_SHA1] == sha1 def _file_helper( self, bucket, sha1_sum=None, bytes_to_write: int | None = None ) -> tuple[DownloadVersion, Sha1HexDigest]: bytes_to_write = bytes_to_write or int(self.info.get_absolute_minimum_part_size()) * 2 + 1 with tempfile.TemporaryDirectory() as temp_dir: temp_dir = pathlib.Path(temp_dir) source_small_file = pathlib.Path(temp_dir) / 'source_small_file' with open(source_small_file, 'wb') as small_file: self.write_zeros(small_file, bytes_to_write) bucket.upload_local_file( source_small_file, 'small_file', sha1_sum=sha1_sum, ) target_small_file = pathlib.Path(temp_dir) / 'target_small_file' f = bucket.download_file_by_name('small_file') f.save_to(target_small_file) source_sha1 = hex_sha1_of_file(source_small_file) assert source_sha1 == hex_sha1_of_file(target_small_file) return f, source_sha1 def test_small(self): bucket = self.create_bucket() f, _ = self._file_helper(bucket, bytes_to_write=1) assert f.download_version.content_sha1_verified def test_small_unverified(self): bucket = self.create_bucket() f, _ = self._file_helper(bucket, sha1_sum='do_not_verify', bytes_to_write=1) if f.download_version.content_sha1_verified: pprint(f.download_version._get_args_for_clone()) assert not f.download_version.content_sha1_verified @pytest.mark.parametrize('size_multiplier', [1, 100]) def test_gzip(b2_auth_data, bucket, tmp_path, b2_api, size_multiplier): """Test downloading gzipped files of varius sizes with and without content-encoding.""" source_file = tmp_path / 'compressed_file.gz' downloaded_uncompressed_file = tmp_path / 'downloaded_uncompressed_file' downloaded_compressed_file = tmp_path / 'downloaded_compressed_file' data_to_write = b"I'm about to be compressed and sent to the cloud, yay!\n" * size_multiplier source_file.write_bytes(gzip.compress(data_to_write)) file_version = bucket.upload_local_file( str(source_file), 'gzipped_file', file_info={'b2-content-encoding': 'gzip'} ) b2_api.download_file_by_id(file_id=file_version.id_).save_to(str(downloaded_compressed_file)) assert downloaded_compressed_file.read_bytes() == source_file.read_bytes() decompressing_api, _ = authorize(b2_auth_data, B2HttpApiConfig(decode_content=True)) decompressing_api.download_file_by_id(file_id=file_version.id_).save_to( str(downloaded_uncompressed_file) ) assert downloaded_uncompressed_file.read_bytes() == data_to_write @pytest.fixture def source_file(tmp_path): source_file = tmp_path / 'source.txt' source_file.write_text('hello world') return source_file @pytest.fixture def uploaded_source_file_version(bucket, source_file): file_version = bucket.upload_local_file(str(source_file), source_file.name) return file_version @pytest.mark.skipif(platform.system() == 'Windows', reason='no os.mkfifo() on Windows') def test_download_to_fifo(bucket, tmp_path, source_file, uploaded_source_file_version, bg_executor): output_file = tmp_path / 'output.txt' os.mkfifo(output_file) output_string = None def reader(): nonlocal output_string output_string = output_file.read_text() reader_future = bg_executor.submit(reader) bucket.download_file_by_id(file_id=uploaded_source_file_version.id_).save_to(output_file) reader_future.result(timeout=1) assert source_file.read_text() == output_string @pytest.fixture def binary_cap(request): """ Get best suited capture. For Windows we need capsys as capfd fails, while on any other (i.e. POSIX systems) we need capfd. This is sadly tied directly to how .save_to() is implemented, as Windows required special handling. """ cap = request.getfixturevalue('capsysbinary' if _IS_WINDOWS else 'capfdbinary') yield cap def test_download_to_stdout(bucket, source_file, uploaded_source_file_version, binary_cap): output_file = 'CON' if _IS_WINDOWS else '/dev/stdout' bucket.download_file_by_id(file_id=uploaded_source_file_version.id_).save_to(output_file) assert binary_cap.readouterr().out == source_file.read_bytes() b2-sdk-python-2.8.0/test/integration/test_file_version_attributes.py000066400000000000000000000040451474454370000260020ustar00rootroot00000000000000###################################################################### # # File: test/integration/test_file_version_attributes.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import datetime as dt from .base import IntegrationTestBase class TestFileVersionAttributes(IntegrationTestBase): def _assert_object_has_attributes(self, object, kwargs): for key, value in kwargs.items(): assert getattr(object, key) == value def test_file_info_b2_attributes(self): # This test checks that attributes that are internally represented as file_info items with prefix `b2-` # are saved and retrieved correctly. bucket = self.create_bucket() expected_attributes = { 'cache_control': 'max-age=3600', 'expires': 'Wed, 21 Oct 2105 07:28:00 GMT', 'content_disposition': 'attachment; filename="fname.ext"', 'content_encoding': 'utf-8', 'content_language': 'en', } kwargs = { **expected_attributes, 'expires': dt.datetime(2105, 10, 21, 7, 28, tzinfo=dt.timezone.utc), } file_version = bucket.upload_bytes(b'0', 'file', **kwargs) self._assert_object_has_attributes(file_version, expected_attributes) file_version = bucket.get_file_info_by_id(file_version.id_) self._assert_object_has_attributes(file_version, expected_attributes) download_file = bucket.download_file_by_id(file_version.id_) self._assert_object_has_attributes(download_file.download_version, expected_attributes) copied_version = bucket.copy( file_version.id_, 'file_copy', content_type='text/plain', **{**kwargs, 'content_language': 'de'}, ) self._assert_object_has_attributes( copied_version, {**expected_attributes, 'content_language': 'de'} ) b2-sdk-python-2.8.0/test/integration/test_raw_api.py000066400000000000000000000614611474454370000224770ustar00rootroot00000000000000###################################################################### # # File: test/integration/test_raw_api.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import io import os import random import re import sys import time import traceback from typing import List import pytest from b2sdk._internal.b2http import B2Http from b2sdk._internal.encryption.setting import ( EncryptionAlgorithm, EncryptionMode, EncryptionSetting, ) from b2sdk._internal.exception import DisablingFileLockNotSupported, Unauthorized from b2sdk._internal.file_lock import ( NO_RETENTION_FILE_SETTING, BucketRetentionSetting, FileRetentionSetting, RetentionMode, RetentionPeriod, ) from b2sdk._internal.raw_api import ( ALL_CAPABILITIES, REALM_URLS, B2RawHTTPApi, NotificationRuleResponse, ) from b2sdk._internal.replication.setting import ReplicationConfiguration, ReplicationRule from b2sdk._internal.replication.types import ReplicationStatus from b2sdk._internal.utils import hex_sha1_of_stream from test.helpers import assert_dict_equal_ignore_extra, type_validator_factory # TODO: rewrite to separate test cases after introduction of reusable bucket def test_raw_api(dont_cleanup_old_buckets): """ Exercise the code in B2RawHTTPApi by making each call once, just to make sure the parameters are passed in, and the result is passed back. The goal is to be a complete test of B2RawHTTPApi, so the tests for the rest of the code can use the simulator. Prints to stdout if things go wrong. :return: 0 on success, non-zero on failure """ application_key_id = os.environ.get('B2_TEST_APPLICATION_KEY_ID') if application_key_id is None: pytest.fail('B2_TEST_APPLICATION_KEY_ID is not set.') application_key = os.environ.get('B2_TEST_APPLICATION_KEY') if application_key is None: pytest.fail('B2_TEST_APPLICATION_KEY is not set.') print() try: raw_api = B2RawHTTPApi(B2Http()) raw_api_test_helper(raw_api, not dont_cleanup_old_buckets) except Exception: traceback.print_exc(file=sys.stdout) pytest.fail('test_raw_api failed') def authorize_raw_api(raw_api): application_key_id = os.environ.get('B2_TEST_APPLICATION_KEY_ID') if application_key_id is None: print('B2_TEST_APPLICATION_KEY_ID is not set.', file=sys.stderr) sys.exit(1) application_key = os.environ.get('B2_TEST_APPLICATION_KEY') if application_key is None: print('B2_TEST_APPLICATION_KEY is not set.', file=sys.stderr) sys.exit(1) realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') realm_url = REALM_URLS.get(realm, realm) auth_dict = raw_api.authorize_account(realm_url, application_key_id, application_key) return auth_dict def raw_api_test_helper(raw_api, should_cleanup_old_buckets): """ Try each of the calls to the raw api. Raise an exception if anything goes wrong. This uses a Backblaze account that is just for this test. The account uses the free level of service, which should be enough to run this test a reasonable number of times each day. If somebody abuses the account for other things, this test will break and we'll have to do something about it. """ # b2_authorize_account print('b2_authorize_account') auth_dict = authorize_raw_api(raw_api) preview_feature_caps = { 'readBucketNotifications', 'writeBucketNotifications', } missing_capabilities = ( set(ALL_CAPABILITIES) - {'readBuckets', 'listAllBucketNames'} - preview_feature_caps - set(auth_dict['apiInfo']['storageApi']['capabilities']) ) assert not missing_capabilities, f'it appears that the raw_api integration test is being run with a non-full key. Missing capabilities: {missing_capabilities}' account_id = auth_dict['accountId'] account_auth_token = auth_dict['authorizationToken'] api_url = auth_dict['apiInfo']['storageApi']['apiUrl'] download_url = auth_dict['apiInfo']['storageApi']['downloadUrl'] # b2_create_key print('b2_create_key') key_dict = raw_api.create_key( api_url, account_auth_token, account_id, ['readFiles'], 'testKey', None, None, None, ) # b2_list_keys print('b2_list_keys') raw_api.list_keys(api_url, account_auth_token, account_id, 10) # b2_delete_key print('b2_delete_key') raw_api.delete_key(api_url, account_auth_token, key_dict['applicationKeyId']) # b2_create_bucket, with a unique bucket name # Include the account ID in the bucket name to be # sure it doesn't collide with bucket names from # other accounts. print('b2_create_bucket') bucket_name = 'test-raw-api-%s-%d-%d' % ( account_id, int(time.time()), random.randint(1000, 9999), ) # very verbose http debug # import http.client; http.client.HTTPConnection.debuglevel = 1 bucket_dict = raw_api.create_bucket( api_url, account_auth_token, account_id, bucket_name, 'allPublic', is_file_lock_enabled=True, ) bucket_id = bucket_dict['bucketId'] first_bucket_revision = bucket_dict['revision'] ################################# print('b2 / replication') # 1) create source key (read permissions) replication_source_key_dict = raw_api.create_key( api_url, account_auth_token, account_id, [ 'listBuckets', 'listFiles', 'readFiles', 'writeFiles', # Pawel @ 2022-06-21: adding this to make tests pass with a weird server validator ], 'testReplicationSourceKey', None, None, None, ) replication_source_key = replication_source_key_dict['applicationKeyId'] # 2) create source bucket with replication to destination - existing bucket try: # in order to test replication, we need to create a second bucket replication_source_bucket_name = 'test-raw-api-%s-%d-%d' % ( account_id, int(time.time()), random.randint(1000, 9999), ) replication_source_bucket_dict = raw_api.create_bucket( api_url, account_auth_token, account_id, replication_source_bucket_name, 'allPublic', is_file_lock_enabled=True, replication=ReplicationConfiguration( rules=[ ReplicationRule( destination_bucket_id=bucket_id, include_existing_files=True, name='test-rule', ), ], source_key_id=replication_source_key, ), ) assert 'replicationConfiguration' in replication_source_bucket_dict assert replication_source_bucket_dict['replicationConfiguration'] == { 'isClientAuthorizedToRead': True, 'value': { 'asReplicationSource': { 'replicationRules': [ { 'destinationBucketId': bucket_id, 'fileNamePrefix': '', 'includeExistingFiles': True, 'isEnabled': True, 'priority': 128, 'replicationRuleName': 'test-rule', }, ], 'sourceApplicationKeyId': replication_source_key, }, 'asReplicationDestination': None, }, } # 3) upload test file and check replication status upload_url_dict = raw_api.get_upload_url( api_url, account_auth_token, replication_source_bucket_dict['bucketId'], ) file_contents = b'hello world' file_dict = raw_api.upload_file( upload_url_dict['uploadUrl'], upload_url_dict['authorizationToken'], 'test.txt', len(file_contents), 'text/plain', hex_sha1_of_stream(io.BytesIO(file_contents), len(file_contents)), {'color': 'blue'}, io.BytesIO(file_contents), ) assert ( ReplicationStatus[file_dict['replicationStatus'].upper()] == ReplicationStatus.PENDING ) finally: raw_api.delete_key(api_url, account_auth_token, replication_source_key) # 4) create destination key (write permissions) replication_destination_key_dict = raw_api.create_key( api_url, account_auth_token, account_id, ['listBuckets', 'listFiles', 'writeFiles'], 'testReplicationDestinationKey', None, None, None, ) replication_destination_key = replication_destination_key_dict['applicationKeyId'] # 5) update destination bucket to receive updates try: bucket_dict = raw_api.update_bucket( api_url, account_auth_token, account_id, bucket_id, 'allPublic', replication=ReplicationConfiguration( source_to_destination_key_mapping={ replication_source_key: replication_destination_key, }, ), ) assert bucket_dict['replicationConfiguration'] == { 'isClientAuthorizedToRead': True, 'value': { 'asReplicationDestination': { 'sourceToDestinationKeyMapping': { replication_source_key: replication_destination_key, }, }, 'asReplicationSource': None, }, } finally: raw_api.delete_key( api_url, account_auth_token, replication_destination_key_dict['applicationKeyId'], ) # 6) cleanup: disable replication for destination and remove source bucket_dict = raw_api.update_bucket( api_url, account_auth_token, account_id, bucket_id, 'allPublic', replication=ReplicationConfiguration(), ) assert bucket_dict['replicationConfiguration'] == { 'isClientAuthorizedToRead': True, 'value': None, } _clean_and_delete_bucket( raw_api, api_url, account_auth_token, account_id, replication_source_bucket_dict['bucketId'], ) ################# print('b2_update_bucket') sse_b2_aes = EncryptionSetting( mode=EncryptionMode.SSE_B2, algorithm=EncryptionAlgorithm.AES256, ) sse_none = EncryptionSetting(mode=EncryptionMode.NONE) for encryption_setting, default_retention in [ ( sse_none, BucketRetentionSetting(mode=RetentionMode.GOVERNANCE, period=RetentionPeriod(days=1)), ), (sse_b2_aes, None), (sse_b2_aes, BucketRetentionSetting(RetentionMode.NONE)), ]: bucket_dict = raw_api.update_bucket( api_url, account_auth_token, account_id, bucket_id, 'allPublic', default_server_side_encryption=encryption_setting, default_retention=default_retention, ) # b2_list_buckets print('b2_list_buckets') bucket_list_dict = raw_api.list_buckets(api_url, account_auth_token, account_id) # print(bucket_list_dict) # b2_get_upload_url print('b2_get_upload_url') upload_url_dict = raw_api.get_upload_url(api_url, account_auth_token, bucket_id) upload_url = upload_url_dict['uploadUrl'] upload_auth_token = upload_url_dict['authorizationToken'] # b2_upload_file print('b2_upload_file') file_name = 'test.txt' file_contents = b'hello world' file_sha1 = hex_sha1_of_stream(io.BytesIO(file_contents), len(file_contents)) file_dict = raw_api.upload_file( upload_url, upload_auth_token, file_name, len(file_contents), 'text/plain', file_sha1, {'color': 'blue', 'b2-cache-control': 'private, max-age=2222'}, io.BytesIO(file_contents), server_side_encryption=sse_b2_aes, # custom_upload_timestamp=12345, file_retention=FileRetentionSetting( RetentionMode.GOVERNANCE, int(time.time() + 100) * 1000, ), ) file_id = file_dict['fileId'] # b2_list_file_versions print('b2_list_file_versions') list_versions_dict = raw_api.list_file_versions(api_url, account_auth_token, bucket_id) assert [file_name] == [f_dict['fileName'] for f_dict in list_versions_dict['files']] assert ['private, max-age=2222'] == [ f_dict['fileInfo']['b2-cache-control'] for f_dict in list_versions_dict['files'] ] # b2_download_file_by_id with auth print('b2_download_file_by_id (auth)') url = raw_api.get_download_url_by_id(download_url, file_id) with raw_api.download_file_from_url(account_auth_token, url) as response: data = next(response.iter_content(chunk_size=len(file_contents))) assert data == file_contents, data # b2_download_file_by_id no auth print('b2_download_file_by_id (no auth)') url = raw_api.get_download_url_by_id(download_url, file_id) with raw_api.download_file_from_url(None, url) as response: data = next(response.iter_content(chunk_size=len(file_contents))) assert data == file_contents, data # b2_download_file_by_name with auth print('b2_download_file_by_name (auth)') url = raw_api.get_download_url_by_name(download_url, bucket_name, file_name) with raw_api.download_file_from_url(account_auth_token, url) as response: data = next(response.iter_content(chunk_size=len(file_contents))) assert data == file_contents, data # b2_download_file_by_name no auth print('b2_download_file_by_name (no auth)') url = raw_api.get_download_url_by_name(download_url, bucket_name, file_name) with raw_api.download_file_from_url(None, url) as response: data = next(response.iter_content(chunk_size=len(file_contents))) assert data == file_contents, data # b2_get_download_authorization print('b2_get_download_authorization') download_auth = raw_api.get_download_authorization( api_url, account_auth_token, bucket_id, file_name[:-2], 12345 ) download_auth_token = download_auth['authorizationToken'] # b2_download_file_by_name with download auth print('b2_download_file_by_name (download auth)') url = raw_api.get_download_url_by_name(download_url, bucket_name, file_name) with raw_api.download_file_from_url(download_auth_token, url) as response: data = next(response.iter_content(chunk_size=len(file_contents))) assert data == file_contents, data # b2_list_file_names print('b2_list_file_names') list_names_dict = raw_api.list_file_names(api_url, account_auth_token, bucket_id) assert [file_name] == [f_dict['fileName'] for f_dict in list_names_dict['files']] # b2_list_file_names (start, count) print('b2_list_file_names (start, count)') list_names_dict = raw_api.list_file_names( api_url, account_auth_token, bucket_id, start_file_name=file_name, max_file_count=5 ) assert [file_name] == [f_dict['fileName'] for f_dict in list_names_dict['files']] # b2_copy_file print('b2_copy_file') copy_file_name = 'test_copy.txt' raw_api.copy_file(api_url, account_auth_token, file_id, copy_file_name) # b2_get_file_info_by_id print('b2_get_file_info_by_id') file_info_dict = raw_api.get_file_info_by_id(api_url, account_auth_token, file_id) assert file_info_dict['fileName'] == file_name # b2_get_file_info_by_name print('b2_get_file_info_by_name (no auth)') info_headers = raw_api.get_file_info_by_name(download_url, None, bucket_name, file_name) assert info_headers['x-bz-file-id'] == file_id # b2_get_file_info_by_name print('b2_get_file_info_by_name (auth)') info_headers = raw_api.get_file_info_by_name( download_url, account_auth_token, bucket_name, file_name ) assert info_headers['x-bz-file-id'] == file_id # b2_get_file_info_by_name print('b2_get_file_info_by_name (download auth)') info_headers = raw_api.get_file_info_by_name( download_url, download_auth_token, bucket_name, file_name ) assert info_headers['x-bz-file-id'] == file_id # b2_hide_file print('b2_hide_file') raw_api.hide_file(api_url, account_auth_token, bucket_id, file_name) # b2_start_large_file print('b2_start_large_file') file_info = {'color': 'red'} large_info = raw_api.start_large_file( api_url, account_auth_token, bucket_id, file_name, 'text/plain', file_info, server_side_encryption=sse_b2_aes, ) large_file_id = large_info['fileId'] # b2_get_upload_part_url print('b2_get_upload_part_url') upload_part_dict = raw_api.get_upload_part_url(api_url, account_auth_token, large_file_id) upload_part_url = upload_part_dict['uploadUrl'] upload_path_auth = upload_part_dict['authorizationToken'] # b2_upload_part print('b2_upload_part') part_contents = b'hello part' part_sha1 = hex_sha1_of_stream(io.BytesIO(part_contents), len(part_contents)) raw_api.upload_part( upload_part_url, upload_path_auth, 1, len(part_contents), part_sha1, io.BytesIO(part_contents), ) # b2_copy_part print('b2_copy_part') raw_api.copy_part(api_url, account_auth_token, file_id, large_file_id, 2, (0, 5)) # b2_list_parts print('b2_list_parts') parts_response = raw_api.list_parts(api_url, account_auth_token, large_file_id, 1, 100) assert [1, 2] == [part['partNumber'] for part in parts_response['parts']] # b2_list_unfinished_large_files unfinished_list = raw_api.list_unfinished_large_files(api_url, account_auth_token, bucket_id) assert [file_name] == [f_dict['fileName'] for f_dict in unfinished_list['files']] assert file_info == unfinished_list['files'][0]['fileInfo'] # b2_finish_large_file print('b2_finish_large_file') try: raw_api.finish_large_file(api_url, account_auth_token, large_file_id, [part_sha1]) raise Exception('finish should have failed') except Exception as e: assert 'large files must have at least 2 parts' in str(e) # TODO: make another attempt to finish but this time successfully # b2_update_bucket print('b2_update_bucket') updated_bucket = raw_api.update_bucket( api_url, account_auth_token, account_id, bucket_id, 'allPrivate', bucket_info={'color': 'blue'}, default_retention=BucketRetentionSetting( mode=RetentionMode.GOVERNANCE, period=RetentionPeriod(days=1) ), is_file_lock_enabled=True, ) assert first_bucket_revision < updated_bucket['revision'] # NOTE: this update_bucket call is only here to be able to find out the error code returned by # the server if an attempt is made to disable file lock. It has to be done here since the CLI # by design does not allow disabling file lock at all (i.e. there is no --fileLockEnabled=false # option or anything equivalent to that). with pytest.raises(DisablingFileLockNotSupported): raw_api.update_bucket( api_url, account_auth_token, account_id, bucket_id, 'allPrivate', is_file_lock_enabled=False, ) # b2_delete_file_version print('b2_delete_file_version') with pytest.raises(Unauthorized): raw_api.delete_file_version(api_url, account_auth_token, file_id, file_name) raw_api.delete_file_version(api_url, account_auth_token, file_id, file_name, True) print('b2_get_bucket_notification_rules & b2_set_bucket_notification_rules') try: _subtest_bucket_notification_rules( raw_api, auth_dict, api_url, account_auth_token, bucket_id ) except pytest.skip.Exception as e: print(e) # Clean up this test. _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id) if should_cleanup_old_buckets: # Clean up from old tests. Empty and delete any buckets more than an hour old. _cleanup_old_buckets(raw_api, auth_dict, bucket_list_dict) def _subtest_bucket_notification_rules(raw_api, auth_dict, api_url, account_auth_token, bucket_id): if 'writeBucketNotifications' not in auth_dict['apiInfo']['storageApi']['capabilities']: pytest.skip('Test account does not have writeBucketNotifications capability') notification_rule = { 'eventTypes': ['b2:ObjectCreated:Copy'], 'isEnabled': False, 'name': 'test-notification-rule', 'objectNamePrefix': 'test/object/prefix/', 'targetConfiguration': { 'targetType': 'webhook', 'url': 'https://example.com/webhook', 'hmacSha256SigningSecret': 'a' * 32, }, } notification_rules_response_list = raw_api.set_bucket_notification_rules( api_url, account_auth_token, bucket_id, [notification_rule] ) notification_rule_response_list_validate = type_validator_factory( List[NotificationRuleResponse] ) notification_rule_response_list_validate(notification_rules_response_list) expected_notification_rule_response_list = [ { **notification_rule, 'isSuspended': False, 'suspensionReason': '', 'targetConfiguration': { **notification_rule['targetConfiguration'], 'customHeaders': None, 'hmacSha256SigningSecret': 'a' * 32, }, } ] assert_dict_equal_ignore_extra( notification_rules_response_list, expected_notification_rule_response_list ) assert raw_api.set_bucket_notification_rules(api_url, account_auth_token, bucket_id, []) == [] assert raw_api.get_bucket_notification_rules(api_url, account_auth_token, bucket_id) == [] def cleanup_old_buckets(): raw_api = B2RawHTTPApi(B2Http()) auth_dict = authorize_raw_api(raw_api) bucket_list_dict = raw_api.list_buckets( auth_dict['apiInfo']['storageApi']['apiUrl'], auth_dict['authorizationToken'], auth_dict['accountId'], ) _cleanup_old_buckets(raw_api, auth_dict, bucket_list_dict) def _cleanup_old_buckets(raw_api, auth_dict, bucket_list_dict): for bucket_dict in bucket_list_dict['buckets']: bucket_id = bucket_dict['bucketId'] bucket_name = bucket_dict['bucketName'] if _should_delete_bucket(bucket_name): print('cleaning up old bucket: ' + bucket_name) _clean_and_delete_bucket( raw_api, auth_dict['apiInfo']['storageApi']['apiUrl'], auth_dict['authorizationToken'], auth_dict['accountId'], bucket_id, ) def _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, bucket_id): # Delete the files. This test never creates more than a few files, # so one call to list_file_versions should get them all. versions_dict = raw_api.list_file_versions(api_url, account_auth_token, bucket_id) for version_dict in versions_dict['files']: file_id = version_dict['fileId'] file_name = version_dict['fileName'] action = version_dict['action'] if action in ['hide', 'upload']: print('b2_delete_file', file_name, action) if ( action == 'upload' and version_dict['fileRetention'] and version_dict['fileRetention']['value']['mode'] is not None ): raw_api.update_file_retention( api_url, account_auth_token, file_id, file_name, NO_RETENTION_FILE_SETTING, bypass_governance=True, ) raw_api.delete_file_version(api_url, account_auth_token, file_id, file_name) else: print('b2_cancel_large_file', file_name) raw_api.cancel_large_file(api_url, account_auth_token, file_id) # Delete the bucket print('b2_delete_bucket', bucket_id) raw_api.delete_bucket(api_url, account_auth_token, account_id, bucket_id) def _should_delete_bucket(bucket_name): # Bucket names for this test look like: c7b22d0b0ad7-1460060364-5670 # Other buckets should not be deleted. match = re.match(r'^test-raw-api-[a-f0-9]+-([0-9]+)-([0-9]+)', bucket_name) if match is None: return False # Is it more than an hour old? bucket_time = int(match.group(1)) now = time.time() return bucket_time + 3600 <= now b2-sdk-python-2.8.0/test/integration/test_sync.py000066400000000000000000000036011474454370000220210ustar00rootroot00000000000000###################################################################### # # File: test/integration/test_sync.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import io import time import pytest from b2sdk.v2 import ( CompareVersionMode, NewerFileSyncMode, Synchronizer, SyncReport, parse_folder, ) @pytest.fixture def local_folder_with_files(tmp_path): folder = tmp_path / 'test' folder.mkdir() (folder / 'a').mkdir() (folder / 'a' / 'foo').write_bytes(b'foo') # space in the name is important as it influences lexicographical sorting used by B2 (folder / 'a b').mkdir() (folder / 'a b' / 'bar').write_bytes(b'bar') return folder def test_sync_folder(b2_api, local_folder_with_files, b2_subfolder): source_folder = parse_folder(str(local_folder_with_files), b2_api) dest_folder = parse_folder(b2_subfolder, b2_api) synchronizer = Synchronizer( max_workers=10, newer_file_mode=NewerFileSyncMode.REPLACE, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=10, # ms ) def sync_and_report(): buf = io.StringIO() reporter = SyncReport(buf, no_progress=True) with reporter: synchronizer.sync_folders( source_folder=source_folder, dest_folder=dest_folder, now_millis=int(1000 * time.time()), reporter=reporter, ) return reporter report = sync_and_report() assert report.total_transfer_files == 2 assert report.total_transfer_bytes == 6 second_sync_report = sync_and_report() assert second_sync_report.total_transfer_files == 0 assert second_sync_report.total_transfer_bytes == 0 b2-sdk-python-2.8.0/test/integration/test_upload.py000066400000000000000000000052021474454370000223300ustar00rootroot00000000000000###################################################################### # # File: test/integration/test_upload.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import io from b2sdk._internal.b2http import B2Http from b2sdk._internal.encryption.setting import EncryptionKey, EncryptionSetting from b2sdk._internal.encryption.types import EncryptionAlgorithm, EncryptionMode from b2sdk.v2 import B2RawHTTPApi from .base import IntegrationTestBase from .test_raw_api import authorize_raw_api class TestUnboundStreamUpload(IntegrationTestBase): def assert_data_uploaded_via_stream(self, data: bytes, part_size: int | None = None): bucket = self.create_bucket() stream = io.BytesIO(data) file_name = 'unbound_stream' bucket.upload_unbound_stream(stream, file_name, recommended_upload_part_size=part_size) downloaded_data = io.BytesIO() bucket.download_file_by_name(file_name).save(downloaded_data) assert downloaded_data.getvalue() == data def test_streamed_small_buffer(self): # 20kb data = b'a small data content' * 1024 self.assert_data_uploaded_via_stream(data) def test_streamed_large_buffer_small_part_size(self): # 10mb data = b'a large data content' * 512 * 1024 # 5mb, the smallest allowed part size self.assert_data_uploaded_via_stream(data, part_size=5 * 1024 * 1024) class TestUploadLargeFile(IntegrationTestBase): def test_ssec_key_id(self): sse_c = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'********************************', key_id='some-id'), ) raw_api = B2RawHTTPApi(B2Http()) auth_dict = authorize_raw_api(raw_api) account_auth_token = auth_dict['authorizationToken'] api_url = auth_dict['apiInfo']['storageApi']['apiUrl'] bucket = self.create_bucket() large_info = raw_api.start_large_file( api_url, account_auth_token, bucket.id_, 'test_largefile_sse_c.txt', 'text/plain', None, server_side_encryption=sse_c, ) assert large_info['fileInfo'] == { 'sse_c_key_id': sse_c.key.key_id, } assert large_info['serverSideEncryption'] == { 'algorithm': 'AES256', 'customerKeyMd5': 'SaaDheEjzuynJH8eW6AEpQ==', 'mode': 'SSE-C', } b2-sdk-python-2.8.0/test/static/000077500000000000000000000000001474454370000164005ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/static/__init__.py000066400000000000000000000005051474454370000205110ustar00rootroot00000000000000###################################################################### # # File: test/static/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/static/test_licenses.py000066400000000000000000000016421474454370000216210ustar00rootroot00000000000000###################################################################### # # File: test/static/test_licenses.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from glob import glob from itertools import islice import pytest def test_files_headers(): for file in glob('**/*.py', recursive=True): with open(file) as fd: file = file.replace( '\\', '/' ) # glob('**/*.py') on Windows returns "b2\bucket.py" (wrong slash) head = ''.join(islice(fd, 9)) if 'All Rights Reserved' not in head: pytest.fail(f'Missing "All Rights Reserved" in the header in: {file}') if file not in head: pytest.fail(f'Wrong file name in the header in: {file}') b2-sdk-python-2.8.0/test/unit/000077500000000000000000000000001474454370000160705ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/__init__.py000066400000000000000000000005031474454370000201770ustar00rootroot00000000000000###################################################################### # # File: test/unit/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/account_info/000077500000000000000000000000001474454370000205375ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/account_info/__init__.py000066400000000000000000000005201474454370000226450ustar00rootroot00000000000000###################################################################### # # File: test/unit/account_info/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/account_info/fixtures.py000066400000000000000000000061061474454370000227650ustar00rootroot00000000000000###################################################################### # # File: test/unit/account_info/fixtures.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from apiver_deps import InMemoryAccountInfo, SqliteAccountInfo from pytest_lazy_fixtures import lf @pytest.fixture def account_info_default_data_schema_0(): return dict( account_id='account_id', auth_token='account_auth', api_url='https://api000.backblazeb2.xyz:8180', download_url='https://f000.backblazeb2.xyz:8180', minimum_part_size=100, application_key='app_key', realm='dev', ) @pytest.fixture def account_info_default_data(account_info_default_data_schema_0, apiver): if apiver in ['v0', 'v1']: return dict( allowed=None, application_key_id='application_key_id', s3_api_url='https://s3.us-west-000.backblazeb2.xyz:8180', **account_info_default_data_schema_0, ) return dict( allowed=None, application_key_id='application_key_id', s3_api_url='https://s3.us-west-000.backblazeb2.xyz:8180', account_id='account_id', auth_token='account_auth', api_url='https://api000.backblazeb2.xyz:8180', download_url='https://f000.backblazeb2.xyz:8180', recommended_part_size=100, absolute_minimum_part_size=50, application_key='app_key', realm='dev', ) @pytest.fixture(scope='session') def in_memory_account_info_factory(): def get_account_info(): return InMemoryAccountInfo() return get_account_info @pytest.fixture def in_memory_account_info(in_memory_account_info_factory): return in_memory_account_info_factory() @pytest.fixture def sqlite_account_info_factory(tmpdir): def get_account_info(file_name=None, schema_0=False): if file_name is None: file_name = str(tmpdir.join('b2_account_info')) if schema_0: last_upgrade_to_run = 0 else: last_upgrade_to_run = None return SqliteAccountInfo(file_name, last_upgrade_to_run) return get_account_info @pytest.fixture def sqlite_account_info(sqlite_account_info_factory): return sqlite_account_info_factory() @pytest.fixture( params=[ lf('in_memory_account_info_factory'), lf('sqlite_account_info_factory'), ] ) def account_info_factory(request): return request.param @pytest.fixture( params=[ lf('in_memory_account_info'), lf('sqlite_account_info'), ] ) def account_info(request): return request.param @pytest.fixture def fake_account_info(mocker): account_info = mocker.MagicMock(name='FakeAccountInfo', spec=InMemoryAccountInfo) account_info.REALM_URLS = { 'dev': 'http://api.backblazeb2.xyz:8180', } account_info.is_same_account.return_value = True account_info.is_same_key.return_value = True return account_info b2-sdk-python-2.8.0/test/unit/account_info/test_account_info.py000066400000000000000000000455521474454370000246320ustar00rootroot00000000000000###################################################################### # # File: test/unit/account_info/test_account_info.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import json import os import platform import shutil import stat import sys import tempfile import unittest.mock as mock from abc import ABCMeta, abstractmethod import pytest from apiver_deps import ( ALL_CAPABILITIES, B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR, AbstractAccountInfo, InMemoryAccountInfo, SqliteAccountInfo, UploadUrlPool, ) from apiver_deps_exception import CorruptAccountInfo, MissingAccountData from .fixtures import * # noqa: F401, F403 class WindowsSafeTempDir(tempfile.TemporaryDirectory): def __exit__(self, exc_type, exc_val, exc_tb): try: super().__exit__(exc_type, exc_val, exc_tb) except OSError: pass class TestAccountInfo: @pytest.fixture(autouse=True) def setup(self, account_info_factory, account_info_default_data): self.account_info_factory = account_info_factory self.account_info_default_data = account_info_default_data @pytest.mark.parametrize( 'application_key_id,realm,expected', ( ('application_key_id', 'dev', True), ('application_key_id', 'test', False), ('different_application_key_id', 'dev', False), ('different_application_key_id', 'test', False), ), ) def test_is_same_key(self, application_key_id, realm, expected): account_info = self.account_info_factory() account_info.set_auth_data(**self.account_info_default_data) assert account_info.is_same_key(application_key_id, realm) is expected @pytest.mark.parametrize( 'account_id,application_key_id,expected', ( ('account_id', 'account_id', True), ('account_id', 'ACCOUNT_ID', False), ('account_id', '123account_id0000000000', True), ('account_id', '234account_id0000000000', True), ('account_id', '123account_id000000000', False), ('account_id', '123account_id0000000001', False), ('account_id', '123account_id00000000000', False), ), ) def test_is_master_key(self, account_id, application_key_id, expected): account_info = self.account_info_factory() account_data = self.account_info_default_data.copy() account_data['account_id'] = account_id account_data['application_key_id'] = application_key_id account_info.set_auth_data(**account_data) assert account_info.is_master_key() is expected, (account_id, application_key_id, expected) @pytest.mark.parametrize( 'account_id,realm,expected', ( ('account_id', 'dev', True), ('account_id', 'test', False), ('different_account_id', 'dev', False), ('different_account_id', 'test', False), ), ) def test_is_same_account(self, account_id, realm, expected): account_info = self.account_info_factory() account_info.set_auth_data(**self.account_info_default_data) assert account_info.is_same_account(account_id, realm) is expected @pytest.mark.parametrize( 's3_api_url', ('https://s3.us-east-123.backblazeb2.com', 'https://s3.us-west-321.backblazeb2.com'), ) def test_s3_api_url(self, s3_api_url): account_info = self.account_info_factory() account_info_default_data = { **self.account_info_default_data, 's3_api_url': s3_api_url, } account_info.set_auth_data(**account_info_default_data) assert s3_api_url == account_info.get_s3_api_url() def test_getting_all_capabilities(self): account_info = self.account_info_factory() assert account_info.all_capabilities() == ALL_CAPABILITIES class TestUploadUrlPool: @pytest.fixture(autouse=True) def setUp(self): self.pool = UploadUrlPool() def test_take_empty(self): assert (None, None) == self.pool.take('a') def test_put_and_take(self): self.pool.put('a', 'url_a1', 'auth_token_a1') self.pool.put('a', 'url_a2', 'auth_token_a2') self.pool.put('b', 'url_b1', 'auth_token_b1') assert ('url_a2', 'auth_token_a2') == self.pool.take('a') assert ('url_a1', 'auth_token_a1') == self.pool.take('a') assert (None, None) == self.pool.take('a') assert ('url_b1', 'auth_token_b1') == self.pool.take('b') assert (None, None) == self.pool.take('b') def test_clear(self): self.pool.put('a', 'url_a1', 'auth_token_a1') self.pool.clear_for_key('a') self.pool.put('b', 'url_b1', 'auth_token_b1') assert (None, None) == self.pool.take('a') assert ('url_b1', 'auth_token_b1') == self.pool.take('b') assert (None, None) == self.pool.take('b') class AccountInfoBase(metaclass=ABCMeta): # it is a mixin to avoid running the tests directly (without inheritance) PERSISTENCE = NotImplemented # subclass should override this @abstractmethod def _make_info(self): """ returns a new object of AccountInfo class which should be tested """ def test_clear(self, account_info_default_data, apiver): account_info = self._make_info() account_info.set_auth_data(**account_info_default_data) account_info.clear() with pytest.raises(MissingAccountData): account_info.get_account_id() with pytest.raises(MissingAccountData): account_info.get_account_auth_token() with pytest.raises(MissingAccountData): account_info.get_api_url() with pytest.raises(MissingAccountData): account_info.get_application_key() with pytest.raises(MissingAccountData): account_info.get_download_url() with pytest.raises(MissingAccountData): account_info.get_realm() with pytest.raises(MissingAccountData): account_info.get_application_key_id() assert not account_info.is_same_key('key_id', 'realm') if apiver in ['v0', 'v1']: with pytest.raises(MissingAccountData): account_info.get_minimum_part_size() else: with pytest.raises(MissingAccountData): account_info.get_recommended_part_size() with pytest.raises(MissingAccountData): account_info.get_absolute_minimum_part_size() def test_set_auth_data_compatibility(self, account_info_default_data): account_info = self._make_info() # The original set_auth_data account_info.set_auth_data(**account_info_default_data) actual = account_info.get_allowed() assert AbstractAccountInfo.DEFAULT_ALLOWED == actual, 'default allowed' # allowed was added later allowed = dict( bucketId=None, bucketName=None, capabilities=['readFiles'], namePrefix=None, ) account_info.set_auth_data( **{ **account_info_default_data, 'allowed': allowed, } ) assert allowed == account_info.get_allowed() def test_clear_bucket_upload_data(self): account_info = self._make_info() account_info.put_bucket_upload_url('bucket-0', 'http://bucket-0', 'bucket-0_auth') account_info.clear_bucket_upload_data('bucket-0') assert (None, None) == account_info.take_bucket_upload_url('bucket-0') def test_large_file_upload_urls(self): account_info = self._make_info() account_info.put_large_file_upload_url('file_0', 'http://file_0', 'auth_0') assert ('http://file_0', 'auth_0') == account_info.take_large_file_upload_url('file_0') assert (None, None) == account_info.take_large_file_upload_url('file_0') def test_clear_large_file_upload_urls(self): account_info = self._make_info() account_info.put_large_file_upload_url('file_0', 'http://file_0', 'auth_0') account_info.clear_large_file_upload_urls('file_0') assert (None, None) == account_info.take_large_file_upload_url('file_0') def test_bucket(self): account_info = self._make_info() bucket = mock.MagicMock() bucket.name = 'my-bucket' bucket.id_ = 'bucket-0' assert account_info.get_bucket_id_or_none_from_bucket_name('my-bucket') is None assert account_info.get_bucket_name_or_none_from_bucket_id('bucket-0') is None account_info.save_bucket(bucket) assert 'bucket-0' == account_info.get_bucket_id_or_none_from_bucket_name('my-bucket') assert 'my-bucket' == account_info.get_bucket_name_or_none_from_bucket_id('bucket-0') if self.PERSISTENCE: assert 'bucket-0' == self._make_info().get_bucket_id_or_none_from_bucket_name( 'my-bucket' ) assert 'my-bucket' == self._make_info().get_bucket_name_or_none_from_bucket_id( 'bucket-0' ) assert ('my-bucket', 'bucket-0') in account_info.list_bucket_names_ids() account_info.remove_bucket_name('my-bucket') assert account_info.get_bucket_id_or_none_from_bucket_name('my-bucket') is None assert account_info.get_bucket_name_or_none_from_bucket_id('bucket-0') is None assert ('my-bucket', 'bucket-0') not in account_info.list_bucket_names_ids() if self.PERSISTENCE: assert self._make_info().get_bucket_id_or_none_from_bucket_name('my-bucket') is None assert self._make_info().get_bucket_name_or_none_from_bucket_id('bucket-0') is None def test_refresh_bucket(self): account_info = self._make_info() assert account_info.get_bucket_id_or_none_from_bucket_name('my-bucket') is None assert account_info.get_bucket_name_or_none_from_bucket_id('a') is None bucket_names = {'a': 'bucket-0', 'b': 'bucket-1'} account_info.refresh_entire_bucket_name_cache(bucket_names.items()) assert 'bucket-0' == account_info.get_bucket_id_or_none_from_bucket_name('a') assert 'a' == account_info.get_bucket_name_or_none_from_bucket_id('bucket-0') if self.PERSISTENCE: assert 'bucket-0' == self._make_info().get_bucket_id_or_none_from_bucket_name('a') assert 'a' == self._make_info().get_bucket_name_or_none_from_bucket_id('bucket-0') @pytest.mark.apiver(to_ver=1) def test_account_info_up_to_v1(self): account_info = self._make_info() account_info.set_auth_data( 'account_id', 'account_auth', 'https://api.backblazeb2.com', 'download_url', 100, 'app_key', 'realm', application_key_id='key_id', ) object_instances = [account_info] if self.PERSISTENCE: object_instances.append(self._make_info()) for info2 in object_instances: assert 'account_id' == info2.get_account_id() assert 'account_auth' == info2.get_account_auth_token() assert 'https://api.backblazeb2.com' == info2.get_api_url() assert 'app_key' == info2.get_application_key() assert 'key_id' == info2.get_application_key_id() assert 'realm' == info2.get_realm() assert 100 == info2.get_minimum_part_size() assert info2.is_same_key('key_id', 'realm') assert not info2.is_same_key('key_id', 'another_realm') assert not info2.is_same_key('another_key_id', 'realm') assert not info2.is_same_key('another_key_id', 'another_realm') @pytest.mark.apiver(from_ver=2) def test_account_info_v2(self): account_info = self._make_info() account_info.set_auth_data( account_id='account_id', auth_token='account_auth', api_url='https://api.backblazeb2.com', download_url='download_url', recommended_part_size=100, absolute_minimum_part_size=50, application_key='app_key', realm='realm', s3_api_url='s3_api_url', allowed=None, application_key_id='key_id', ) object_instances = [account_info] if self.PERSISTENCE: object_instances.append(self._make_info()) for info2 in object_instances: assert 'account_id' == info2.get_account_id() assert 'account_auth' == info2.get_account_auth_token() assert 'https://api.backblazeb2.com' == info2.get_api_url() assert 'app_key' == info2.get_application_key() assert 'key_id' == info2.get_application_key_id() assert 'realm' == info2.get_realm() assert 100 == info2.get_recommended_part_size() assert 50 == info2.get_absolute_minimum_part_size() assert info2.is_same_key('key_id', 'realm') assert not info2.is_same_key('key_id', 'another_realm') assert not info2.is_same_key('another_key_id', 'realm') assert not info2.is_same_key('another_key_id', 'another_realm') class TestInMemoryAccountInfo(AccountInfoBase): PERSISTENCE = False def _make_info(self): return InMemoryAccountInfo() class TestSqliteAccountInfo(AccountInfoBase): PERSISTENCE = True @pytest.fixture(autouse=True) def setUp(self, request): self.db_path = tempfile.NamedTemporaryFile( prefix=f'tmp_b2_tests_{request.node.name}__', delete=True ).name try: os.unlink(self.db_path) except OSError: pass self.test_home = tempfile.mkdtemp() yield for cleanup_method in [ lambda: os.unlink(self.db_path), lambda: shutil.rmtree(self.test_home), ]: try: cleanup_method() except OSError: pass @pytest.mark.skipif( platform.system() == 'Windows', reason='different permission system on Windows', ) def test_permissions(self): """ Test that a new database won't be readable by just any user """ SqliteAccountInfo( file_name=self.db_path, ) mode = os.stat(self.db_path).st_mode assert stat.filemode(mode) == '-rw-------' def test_corrupted(self): """ Test that a corrupted file will be replaced with a blank file. """ with open(self.db_path, 'wb') as f: f.write(b'not a valid database') with pytest.raises(CorruptAccountInfo): self._make_info() @pytest.mark.skipif( platform.system() == 'Windows', reason='it fails to upgrade on Windows, not worth to fix it anymore', ) def test_convert_from_json(self): """ Tests converting from a JSON account info file, which is what version 0.5.2 of the command-line tool used. """ data = dict( account_auth_token='auth_token', account_id='account_id', api_url='api_url', application_key='application_key', download_url='download_url', minimum_part_size=5000, realm='production', ) with open(self.db_path, 'wb') as f: f.write(json.dumps(data).encode('utf-8')) account_info = self._make_info() assert 'auth_token' == account_info.get_account_auth_token() def _make_info(self): return self._make_sqlite_account_info() def _make_sqlite_account_info(self, env=None, last_upgrade_to_run=None): """ Returns a new SqliteAccountInfo that has just read the data from the file. :param dict env: Override Environment variables. """ # Override HOME to ensure hermetic tests with mock.patch('os.environ', env or {'HOME': self.test_home}): return SqliteAccountInfo( file_name=self.db_path if not env else None, last_upgrade_to_run=last_upgrade_to_run, ) def test_uses_xdg_config_home(self, apiver): is_xdg_os = bool(SqliteAccountInfo._get_xdg_config_path()) with WindowsSafeTempDir() as d: env = { 'HOME': self.test_home, 'USERPROFILE': self.test_home, } if is_xdg_os: # pass the env. variable on XDG-like OS only env[XDG_CONFIG_HOME_ENV_VAR] = d account_info = self._make_sqlite_account_info(env=env) if apiver in ['v0', 'v1']: expected_path = os.path.abspath(os.path.join(self.test_home, '.b2_account_info')) elif is_xdg_os: assert os.path.exists(os.path.join(d, 'b2')) expected_path = os.path.abspath(os.path.join(d, 'b2', 'account_info')) else: expected_path = os.path.abspath(os.path.join(self.test_home, '.b2_account_info')) actual_path = os.path.abspath(account_info.filename) assert expected_path == actual_path def test_uses_existing_file_and_ignores_xdg(self): with WindowsSafeTempDir() as d: default_db_file_location = os.path.join(self.test_home, '.b2_account_info') open(default_db_file_location, 'a').close() account_info = self._make_sqlite_account_info( env={ 'HOME': self.test_home, 'USERPROFILE': self.test_home, XDG_CONFIG_HOME_ENV_VAR: d, } ) actual_path = os.path.abspath(account_info.filename) assert default_db_file_location == actual_path assert not os.path.exists(os.path.join(d, 'b2')) def test_account_info_env_var_overrides_xdg_config_home(self): with WindowsSafeTempDir() as d: account_info = self._make_sqlite_account_info( env={ 'HOME': self.test_home, 'USERPROFILE': self.test_home, XDG_CONFIG_HOME_ENV_VAR: d, B2_ACCOUNT_INFO_ENV_VAR: os.path.join(d, 'b2_account_info'), } ) expected_path = os.path.abspath(os.path.join(d, 'b2_account_info')) actual_path = os.path.abspath(account_info.filename) assert expected_path == actual_path def test_resolve_xdg_os_default(self): is_xdg_os = bool(SqliteAccountInfo._get_xdg_config_path()) assert is_xdg_os == (sys.platform not in ('win32', 'darwin')) def test_resolve_xdg_os_default_no_env_var(self, monkeypatch): # ensure that XDG_CONFIG_HOME_ENV_VAR doesn't to resolve XDG-like OS monkeypatch.delenv(XDG_CONFIG_HOME_ENV_VAR, raising=False) is_xdg_os = bool(SqliteAccountInfo._get_xdg_config_path()) assert is_xdg_os == (sys.platform not in ('win32', 'darwin')) b2-sdk-python-2.8.0/test/unit/account_info/test_sqlite_account_info.py000066400000000000000000000147141474454370000262070ustar00rootroot00000000000000###################################################################### # # File: test/unit/account_info/test_sqlite_account_info.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import os import pytest from apiver_deps import ( B2_ACCOUNT_INFO_DEFAULT_FILE, B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR, AbstractAccountInfo, SqliteAccountInfo, ) from .fixtures import * class TestDatabseMigrations: @pytest.fixture(autouse=True) def setup(self, sqlite_account_info_factory, account_info_default_data_schema_0): self.sqlite_account_info_factory = sqlite_account_info_factory self.account_info_default_data = account_info_default_data_schema_0 def test_upgrade_1_default_allowed(self): """The 'allowed' field should be the default for upgraded databases.""" old_account_info = self.sqlite_account_info_factory(schema_0=True) old_account_info.set_auth_data_with_schema_0_for_test(**self.account_info_default_data) new_account_info = self.sqlite_account_info_factory(file_name=old_account_info.filename) assert AbstractAccountInfo.DEFAULT_ALLOWED == new_account_info.get_allowed() def test_upgrade_2_default_app_key(self): """The 'application_key_id' field should default to the account ID.""" old_account_info = self.sqlite_account_info_factory(schema_0=True) old_account_info.set_auth_data_with_schema_0_for_test(**self.account_info_default_data) new_account_info = self.sqlite_account_info_factory(file_name=old_account_info.filename) assert 'account_id' == new_account_info.get_application_key_id() def test_upgrade_3_default_s3_api_url(self): """The 's3_api_url' field should be set.""" old_account_info = self.sqlite_account_info_factory(schema_0=True) old_account_info.set_auth_data_with_schema_0_for_test(**self.account_info_default_data) new_account_info = self.sqlite_account_info_factory(file_name=old_account_info.filename) assert '' == new_account_info.get_s3_api_url() def test_migrate_to_4(self): old_account_info = self.sqlite_account_info_factory(schema_0=True) old_account_info.set_auth_data_with_schema_0_for_test(**self.account_info_default_data) new_account_info = self.sqlite_account_info_factory(file_name=old_account_info.filename) with new_account_info._get_connection() as conn: sizes = conn.execute( 'SELECT recommended_part_size, absolute_minimum_part_size from account' ).fetchone() assert (100, 5000000) == sizes class TestSqliteAccountProfileFileLocation: @pytest.fixture(autouse=True) def setup(self, monkeypatch, tmpdir): monkeypatch.setenv( 'HOME', str(tmpdir) ) # this affects .expanduser() and protects the real HOME folder monkeypatch.setenv('USERPROFILE', str(tmpdir)) # same as HOME, but for Windows monkeypatch.delenv(B2_ACCOUNT_INFO_ENV_VAR, raising=False) monkeypatch.delenv(XDG_CONFIG_HOME_ENV_VAR, raising=False) def test_invalid_profile_name(self): with pytest.raises(ValueError): SqliteAccountInfo.get_user_account_info_path(profile='&@(*$') def test_profile_and_file_name_conflict(self): with pytest.raises(ValueError): SqliteAccountInfo.get_user_account_info_path(file_name='foo', profile='bar') def test_profile_and_env_var_conflict(self, monkeypatch): monkeypatch.setenv(B2_ACCOUNT_INFO_ENV_VAR, 'foo') with pytest.raises(ValueError): SqliteAccountInfo.get_user_account_info_path(profile='bar') def test_profile_and_xdg_config_env_var(self, monkeypatch): monkeypatch.setenv(XDG_CONFIG_HOME_ENV_VAR, os.path.join('~', 'custom')) account_info_path = SqliteAccountInfo.get_user_account_info_path(profile='secondary') assert account_info_path == os.path.expanduser( os.path.join('~', 'custom', 'b2', 'db-secondary.sqlite') ) def test_profile(self, monkeypatch): xdg_config_path = SqliteAccountInfo._get_xdg_config_path() if xdg_config_path: expected_path = (xdg_config_path, 'b2', 'db-foo.sqlite') else: expected_path = ('~', '.b2db-foo.sqlite') account_info_path = SqliteAccountInfo.get_user_account_info_path(profile='foo') assert account_info_path == os.path.expanduser(os.path.join(*expected_path)) def test_file_name(self): account_info_path = SqliteAccountInfo.get_user_account_info_path( file_name=os.path.join('~', 'foo') ) assert account_info_path == os.path.expanduser(os.path.join('~', 'foo')) def test_env_var(self, monkeypatch): monkeypatch.setenv(B2_ACCOUNT_INFO_ENV_VAR, os.path.join('~', 'foo')) account_info_path = SqliteAccountInfo.get_user_account_info_path() assert account_info_path == os.path.expanduser(os.path.join('~', 'foo')) def test_default_file_if_exists(self, monkeypatch): # ensure that XDG_CONFIG_HOME_ENV_VAR doesn't matter if default file exists monkeypatch.setenv(XDG_CONFIG_HOME_ENV_VAR, 'some') account_file_path = os.path.expanduser(B2_ACCOUNT_INFO_DEFAULT_FILE) parent_dir = os.path.abspath(os.path.join(account_file_path, os.pardir)) os.makedirs(parent_dir, exist_ok=True) with open(account_file_path, 'w') as account_file: account_file.write('') account_info_path = SqliteAccountInfo.get_user_account_info_path() assert account_info_path == os.path.expanduser(B2_ACCOUNT_INFO_DEFAULT_FILE) def test_xdg_config_env_var(self, monkeypatch): monkeypatch.setenv(XDG_CONFIG_HOME_ENV_VAR, os.path.join('~', 'custom')) account_info_path = SqliteAccountInfo.get_user_account_info_path() assert account_info_path == os.path.expanduser( os.path.join('~', 'custom', 'b2', 'account_info') ) def test_default_file(self): xdg_config_path = SqliteAccountInfo._get_xdg_config_path() if xdg_config_path: expected_path = os.path.join(xdg_config_path, 'b2', 'account_info') else: expected_path = B2_ACCOUNT_INFO_DEFAULT_FILE account_info_path = SqliteAccountInfo.get_user_account_info_path() assert account_info_path == os.path.expanduser(expected_path) b2-sdk-python-2.8.0/test/unit/api/000077500000000000000000000000001474454370000166415ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/api/__init__.py000066400000000000000000000005071474454370000207540ustar00rootroot00000000000000###################################################################### # # File: test/unit/api/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/api/test_api.py000066400000000000000000000642111474454370000210270ustar00rootroot00000000000000###################################################################### # # File: test/unit/api/test_api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import time from contextlib import suppress from unittest import mock import apiver_deps import pytest from apiver_deps import ( NO_RETENTION_FILE_SETTING, ApplicationKey, B2Api, B2Http, B2HttpApiConfig, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, FileIdAndName, FileRetentionSetting, FullApplicationKey, InMemoryAccountInfo, InMemoryCache, LegalHold, RawSimulator, RetentionMode, ) from apiver_deps_exception import AccessDenied, FileNotPresent, InvalidArgument, RestrictedBucket from ..test_base import create_key if apiver_deps.V <= 1: from apiver_deps import FileVersionInfo as VFileVersion else: from apiver_deps import FileVersion as VFileVersion class TestApi: @pytest.fixture(autouse=True) def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = InMemoryCache() self.api = B2Api( self.account_info, self.cache, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator) ) self.raw_api = self.api.session.raw_api (self.application_key_id, self.master_key) = self.raw_api.create_account() @pytest.mark.apiver(from_ver=3) def test_authorize_signature_v3_default_realm(self): self.api.authorize_account( self.application_key_id, self.master_key, ) @pytest.mark.apiver(from_ver=3) def test_authorize_signature_v3_explicit_realm(self): self.api.authorize_account( self.application_key_id, self.master_key, 'production', ) @pytest.mark.apiver(to_ver=2) def test_authorize_signature_up_to_ver_2(self): self.api.authorize_account( 'production', self.application_key_id, self.master_key, ) def test_get_file_info(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') created_file = bucket.upload_bytes( b'hello world', 'file', cache_control='private, max-age=3600' ) result = self.api.get_file_info(created_file.id_) if apiver_deps.V <= 1: self.maxDiff = None with suppress(KeyError): del result['replicationStatus'] assert result == { 'accountId': 'account-0', 'action': 'upload', 'bucketId': 'bucket_0', 'contentLength': 11, 'contentSha1': '2aae6c35c94fcfb415dbe95f408b9ce91ee846ed', 'contentType': 'b2/x-auto', 'fileId': '9999', 'fileInfo': { 'b2-cache-control': 'private, max-age=3600', }, 'fileName': 'file', 'fileRetention': {'isClientAuthorizedToRead': True, 'value': {'mode': None}}, 'legalHold': {'isClientAuthorizedToRead': True, 'value': None}, 'serverSideEncryption': {'mode': 'none'}, 'uploadTimestamp': 5000, } else: assert isinstance(result, VFileVersion) assert result == created_file def test_get_file_info_by_name(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') bucket.upload_bytes( b'hello world', 'file', ) result = self.api.get_file_info_by_name('bucket1', 'file') expected_result = { 'fileId': '9999', 'fileName': 'file', 'fileInfo': {}, 'serverSideEncryption': {'mode': 'none'}, 'legalHold': None, 'fileRetention': {'mode': None, 'retainUntilTimestamp': None}, 'size': 11, 'uploadTimestamp': 5000, 'contentType': 'b2/x-auto', 'contentSha1': '2aae6c35c94fcfb415dbe95f408b9ce91ee846ed', 'replicationStatus': None, } if apiver_deps.V <= 1: expected_result.update( { 'accountId': None, 'action': 'upload', 'bucketId': None, } ) assert result.as_dict() == expected_result def test_get_hidden_file_info_by_name(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') bucket.upload_bytes( b'hello world', 'hidden-file.txt', ) bucket.hide_file('hidden-file.txt') result = self.api.get_file_info_by_name('bucket1', 'hidden-file.txt') expected_result = { 'fileId': '9998', 'fileName': 'hidden-file.txt', 'fileInfo': {}, 'serverSideEncryption': {'mode': 'none'}, 'legalHold': None, 'fileRetention': {'mode': None, 'retainUntilTimestamp': None}, 'size': 0, 'uploadTimestamp': 5001, 'contentSha1': 'none', 'replicationStatus': None, } if apiver_deps.V <= 1: expected_result.update( { 'accountId': None, 'action': 'upload', 'bucketId': None, } ) assert result.as_dict() == expected_result def test_get_non_existent_file_info_by_name(self): self._authorize_account() _ = self.api.create_bucket('bucket1', 'allPrivate') with pytest.raises(FileNotPresent): _ = self.api.get_file_info_by_name('bucket1', 'file-does-not-exist.txt') def test_get_file_info_by_name_with_properties(self): encr_setting = EncryptionSetting( EncryptionMode.SSE_C, EncryptionAlgorithm.AES256, EncryptionKey(b'secret', None) ) lh_setting = LegalHold.ON retention_setting = FileRetentionSetting(RetentionMode.COMPLIANCE, 100) self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate', is_file_lock_enabled=True) _ = bucket.upload_bytes( b'hello world', 'file', encryption=encr_setting, legal_hold=lh_setting, file_retention=retention_setting, ) result = self.api.get_file_info_by_name('bucket1', 'file') assert encr_setting.mode == result.server_side_encryption.mode assert encr_setting.algorithm == result.server_side_encryption.algorithm assert lh_setting == result.legal_hold assert retention_setting == result.file_retention @pytest.mark.parametrize( 'expected_delete_bucket_output', [ pytest.param(None, marks=pytest.mark.apiver(from_ver=1)), pytest.param( { 'accountId': 'account-0', 'bucketName': 'bucket2', 'bucketId': 'bucket_1', 'bucketType': 'allPrivate', 'bucketInfo': {}, 'corsRules': [], 'lifecycleRules': [], 'options': set(), 'revision': 1, 'defaultServerSideEncryption': { 'isClientAuthorizedToRead': True, 'value': {'mode': 'none'}, }, 'fileLockConfiguration': { 'isClientAuthorizedToRead': True, 'value': { 'defaultRetention': {'mode': None, 'period': None}, 'isFileLockEnabled': None, }, }, }, marks=pytest.mark.apiver(to_ver=0), ), ], ) def test_list_buckets(self, expected_delete_bucket_output): self._authorize_account() self.api.create_bucket('bucket1', 'allPrivate') bucket2 = self.api.create_bucket('bucket2', 'allPrivate') delete_output = self.api.delete_bucket(bucket2) if expected_delete_bucket_output is not None: with suppress(KeyError): del delete_output['replicationConfiguration'] assert delete_output == expected_delete_bucket_output self.api.create_bucket('bucket3', 'allPrivate') assert [b.name for b in self.api.list_buckets()] == ['bucket1', 'bucket3'] def test_list_buckets_with_name(self): self._authorize_account() self.api.create_bucket('bucket1', 'allPrivate') self.api.create_bucket('bucket2', 'allPrivate') assert [b.name for b in self.api.list_buckets(bucket_name='bucket1')] == ['bucket1'] @pytest.mark.apiver(from_ver=3) def test_list_buckets_from_cache(self): bucket = type('bucket', (), {'name': 'bucket', 'id_': 'ID-0'}) self._authorize_account() self.cache.set_bucket_name_cache([bucket]) def list_buckets(*args, **kwargs): buckets = self.api.list_buckets(*args, **kwargs) return [(b.name, b.id_) for b in buckets] assert list_buckets(use_cache=True) == [('bucket', 'ID-0')] assert list_buckets(bucket_name='bucket', use_cache=True) == [('bucket', 'ID-0')] assert list_buckets(bucket_name='bucket2', use_cache=True) == [] assert list_buckets(bucket_id='ID-0', use_cache=True) == [('bucket', 'ID-0')] assert list_buckets(bucket_id='ID-2', use_cache=True) == [] assert self.api.list_buckets() == [] def test_buckets_with_encryption(self): self._authorize_account() sse_b2_aes = EncryptionSetting( mode=EncryptionMode.SSE_B2, algorithm=EncryptionAlgorithm.AES256, ) no_encryption = EncryptionSetting( mode=EncryptionMode.NONE, ) unknown_encryption = EncryptionSetting( mode=EncryptionMode.UNKNOWN, ) b1 = self.api.create_bucket( 'bucket1', 'allPrivate', default_server_side_encryption=sse_b2_aes, ) self._verify_if_bucket_is_encrypted(b1, should_be_encrypted=True) b2 = self.api.create_bucket('bucket2', 'allPrivate') self._verify_if_bucket_is_encrypted(b2, should_be_encrypted=False) # uses list_buckets self._check_if_bucket_is_encrypted('bucket1', should_be_encrypted=True) self._check_if_bucket_is_encrypted('bucket2', should_be_encrypted=False) # update to set encryption on b2 b2.update(default_server_side_encryption=sse_b2_aes) self._check_if_bucket_is_encrypted('bucket1', should_be_encrypted=True) self._check_if_bucket_is_encrypted('bucket2', should_be_encrypted=True) # update to unset encryption again b2.update(default_server_side_encryption=no_encryption) self._check_if_bucket_is_encrypted('bucket1', should_be_encrypted=True) self._check_if_bucket_is_encrypted('bucket2', should_be_encrypted=False) # now check it with no readBucketEncryption permission to see that it's unknown key = create_key(self.api, ['listBuckets'], 'key1') self.api.authorize_account( application_key_id=key.id_, application_key=key.application_key, realm='production', ) buckets = { b.name: b for b in self.api.list_buckets() # scan again with new key } assert buckets['bucket1'].default_server_side_encryption == unknown_encryption assert buckets['bucket2'].default_server_side_encryption == unknown_encryption def _check_if_bucket_is_encrypted(self, bucket_name, should_be_encrypted): buckets = {b.name: b for b in self.api.list_buckets()} bucket = buckets[bucket_name] return self._verify_if_bucket_is_encrypted(bucket, should_be_encrypted) def _verify_if_bucket_is_encrypted(self, bucket, should_be_encrypted): sse_b2_aes = EncryptionSetting( mode=EncryptionMode.SSE_B2, algorithm=EncryptionAlgorithm.AES256, ) no_encryption = EncryptionSetting( mode=EncryptionMode.NONE, ) if not should_be_encrypted: assert bucket.default_server_side_encryption == no_encryption else: assert bucket.default_server_side_encryption == sse_b2_aes assert bucket.default_server_side_encryption.mode == EncryptionMode.SSE_B2 assert bucket.default_server_side_encryption.algorithm == EncryptionAlgorithm.AES256 def test_list_buckets_with_id(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') self.api.create_bucket('bucket2', 'allPrivate') assert [b.name for b in self.api.list_buckets(bucket_id=bucket.id_)] == ['bucket1'] def test_reauthorize_with_app_key(self): # authorize and create a key self._authorize_account() key = create_key(self.api, ['listBuckets'], 'key1') # authorize with the key self.api.authorize_account( application_key_id=key.id_, application_key=key.application_key, realm='production', ) # expire the auth token we just got self.raw_api.expire_auth_token(self.account_info.get_account_auth_token()) # listing buckets should work, after it re-authorizes self.api.list_buckets() def test_list_buckets_with_restriction(self): self._authorize_account() bucket1 = self.api.create_bucket('bucket1', 'allPrivate') self.api.create_bucket('bucket2', 'allPrivate') key = create_key(self.api, ['listBuckets'], 'key1', bucket_id=bucket1.id_) self.api.authorize_account( application_key_id=key.id_, application_key=key.application_key, realm='production', ) assert [b.name for b in self.api.list_buckets(bucket_name=bucket1.name)] == ['bucket1'] def test_get_bucket_by_name_with_bucket_restriction(self): self._authorize_account() bucket1 = self.api.create_bucket('bucket1', 'allPrivate') key = create_key(self.api, ['listBuckets'], 'key1', bucket_id=bucket1.id_) self.api.authorize_account( application_key_id=key.id_, application_key=key.application_key, realm='production', ) assert self.api.get_bucket_by_name('bucket1').id_ == bucket1.id_ def test_list_buckets_with_restriction_and_wrong_name(self): self._authorize_account() bucket1 = self.api.create_bucket('bucket1', 'allPrivate') bucket2 = self.api.create_bucket('bucket2', 'allPrivate') key = create_key(self.api, ['listBuckets'], 'key1', bucket_id=bucket1.id_) self.api.authorize_account( application_key_id=key.id_, application_key=key.application_key, realm='production', ) with pytest.raises(RestrictedBucket) as excinfo: self.api.list_buckets(bucket_name=bucket2.name) assert str(excinfo.value) == 'Application key is restricted to bucket: bucket1' def test_list_buckets_with_restriction_and_no_name(self): self._authorize_account() bucket1 = self.api.create_bucket('bucket1', 'allPrivate') self.api.create_bucket('bucket2', 'allPrivate') key = create_key(self.api, ['listBuckets'], 'key1', bucket_id=bucket1.id_) self.api.authorize_account( application_key_id=key.id_, application_key=key.application_key, realm='production', ) with pytest.raises(RestrictedBucket) as excinfo: self.api.list_buckets() assert str(excinfo.value) == 'Application key is restricted to bucket: bucket1' def test_list_buckets_with_restriction_and_wrong_id(self): self._authorize_account() bucket1 = self.api.create_bucket('bucket1', 'allPrivate') self.api.create_bucket('bucket2', 'allPrivate') key = create_key(self.api, ['listBuckets'], 'key1', bucket_id=bucket1.id_) self.api.authorize_account( application_key_id=key.id_, application_key=key.application_key, realm='production', ) with pytest.raises(RestrictedBucket) as excinfo: self.api.list_buckets(bucket_id='not the one bound to the key') assert str(excinfo.value) == f'Application key is restricted to bucket: {bucket1.id_}' def _authorize_account(self): self.api.authorize_account( application_key_id=self.application_key_id, application_key=self.master_key, realm='production', ) def test_update_file_retention(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate', is_file_lock_enabled=True) created_file = bucket.upload_bytes(b'hello world', 'file') assert created_file.file_retention == NO_RETENTION_FILE_SETTING new_retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 100) read_file_retention = self.api.update_file_retention( created_file.id_, created_file.file_name, new_retention ) assert new_retention == read_file_retention if apiver_deps.V <= 1: file_version = bucket.get_file_info_by_id(created_file.id_) else: file_version = self.api.get_file_info(created_file.id_) assert new_retention == file_version.file_retention def test_update_legal_hold(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate', is_file_lock_enabled=True) created_file = bucket.upload_bytes(b'hello world', 'file') assert created_file.legal_hold == LegalHold.UNSET new_legal_hold = LegalHold.ON read_legal_hold = self.api.update_file_legal_hold( created_file.id_, created_file.file_name, new_legal_hold ) assert new_legal_hold == read_legal_hold if apiver_deps.V <= 1: file_version = bucket.get_file_info_by_id(created_file.id_) else: file_version = self.api.get_file_info(created_file.id_) assert new_legal_hold == file_version.legal_hold @pytest.mark.apiver(from_ver=2) def test_cancel_large_file_v2(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') unfinished_large_file = self.api.services.large_file.start_large_file( bucket.id_, 'a_large_file' ) cancel_result = self.api.cancel_large_file(unfinished_large_file.file_id) assert cancel_result == FileIdAndName('9999', 'a_large_file') @pytest.mark.apiver(to_ver=1) def test_cancel_large_file_v1(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') unfinished_large_file = self.api.services.large_file.start_large_file( bucket.id_, 'a_large_file' ) cancel_result = self.api.cancel_large_file(unfinished_large_file.file_id) assert cancel_result == VFileVersion( id_='9999', file_name='a_large_file', size=0, content_type='unknown', content_sha1='none', file_info={}, upload_timestamp=0, action='cancel', api=self.api, ) @pytest.mark.apiver(to_ver=1) def test_provide_raw_api_v1(self): from apiver_deps import B2RawApi # test for legacy name old_style_api = B2Api(raw_api=B2RawApi(B2Http(user_agent_append='test append'))) new_style_api = B2Api(api_config=B2HttpApiConfig(user_agent_append='test append')) assert ( old_style_api.session.raw_api.b2_http.user_agent == new_style_api.session.raw_api.b2_http.user_agent ) with pytest.raises(InvalidArgument): B2Api( raw_api=B2RawApi(B2Http(user_agent_append='test append')), api_config=B2HttpApiConfig(user_agent_append='test append'), ) @pytest.mark.apiver(to_ver=1) def test_create_and_delete_key_v1(self): self._authorize_account() create_result = self.api.create_key(['readFiles'], 'testkey') assert create_result == { 'accountId': self.account_info.get_account_id(), 'applicationKey': 'appKey0', 'applicationKeyId': 'appKeyId0', 'capabilities': ['readFiles'], 'keyName': 'testkey', } delete_result = self.api.delete_key(create_result['applicationKeyId']) self.assertDeleteAndCreateResult(create_result, delete_result) create_result = self.api.create_key(['readFiles'], 'testkey') delete_result = self.api.delete_key_by_id(create_result['applicationKeyId']) self.assertDeleteAndCreateResult(create_result, delete_result.as_dict()) @pytest.mark.apiver(from_ver=2) def test_create_and_delete_key_v2(self): self._authorize_account() bucket = self.api.create_bucket('bucket', 'allPrivate') now = time.time() create_result = self.api.create_key( ['readFiles'], 'testkey', valid_duration_seconds=100, bucket_id=bucket.id_, name_prefix='name', ) assert isinstance(create_result, FullApplicationKey) assert create_result.key_name == 'testkey' assert create_result.capabilities == ['readFiles'] assert create_result.account_id == self.account_info.get_account_id() assert ( (now + 100 - 10) * 1000 < create_result.expiration_timestamp_millis < (now + 100 + 10) * 1000 ) assert create_result.bucket_id == bucket.id_ assert create_result.name_prefix == 'name' # assert create_result.options == ... TODO delete_result = self.api.delete_key(create_result) self.assertDeleteAndCreateResult(create_result, delete_result) create_result = self.api.create_key( ['readFiles'], 'testkey', valid_duration_seconds=100, bucket_id=bucket.id_, name_prefix='name', ) delete_result = self.api.delete_key_by_id(create_result.id_) self.assertDeleteAndCreateResult(create_result, delete_result) def assertDeleteAndCreateResult(self, create_result, delete_result): if apiver_deps.V <= 1: create_result.pop('applicationKey') assert delete_result == create_result else: assert isinstance(delete_result, ApplicationKey) assert delete_result.key_name == create_result.key_name assert delete_result.capabilities == create_result.capabilities assert delete_result.account_id == create_result.account_id assert ( delete_result.expiration_timestamp_millis == create_result.expiration_timestamp_millis ) assert delete_result.bucket_id == create_result.bucket_id assert delete_result.name_prefix == create_result.name_prefix @pytest.mark.apiver(to_ver=1) def test_list_keys_v1(self): self._authorize_account() for i in range(20): self.api.create_key(['readFiles'], f'testkey{i}') with mock.patch.object(self.api, 'DEFAULT_LIST_KEY_COUNT', 10): response = self.api.list_keys() assert response['nextApplicationKeyId'] == 'appKeyId18' assert response['keys'] == [ { 'accountId': 'account-0', 'applicationKeyId': f'appKeyId{ind}', 'bucketId': None, 'capabilities': ['readFiles'], 'expirationTimestamp': None, 'keyName': f'testkey{ind}', 'namePrefix': None, } for ind in [ 0, 1, 10, 11, 12, 13, 14, 15, 16, 17, ] ] @pytest.mark.apiver(from_ver=2) def test_list_keys_v2(self): self._authorize_account() for i in range(20): self.api.create_key(['readFiles'], f'testkey{i}') with mock.patch.object(self.api, 'DEFAULT_LIST_KEY_COUNT', 10): keys = list(self.api.list_keys()) assert [key.id_ for key in keys] == [ 'appKeyId0', 'appKeyId1', 'appKeyId10', 'appKeyId11', 'appKeyId12', 'appKeyId13', 'appKeyId14', 'appKeyId15', 'appKeyId16', 'appKeyId17', 'appKeyId18', 'appKeyId19', 'appKeyId2', 'appKeyId3', 'appKeyId4', 'appKeyId5', 'appKeyId6', 'appKeyId7', 'appKeyId8', 'appKeyId9', ] assert isinstance(keys[0], ApplicationKey) def test_get_key(self): self._authorize_account() key = self.api.create_key(['readFiles'], 'testkey') if apiver_deps.V <= 1: key_id = key['applicationKeyId'] else: key_id = key.id_ assert self.api.get_key(key_id) is not None if apiver_deps.V <= 1: self.api.delete_key(key_id) else: self.api.delete_key(key) assert self.api.get_key(key_id) is None assert self.api.get_key('non-existent') is None def test_delete_file_version_bypass_governance(self): self._authorize_account() bucket = self.api.create_bucket('bucket1', 'allPrivate') created_file = bucket.upload_bytes( b'hello world', 'file', file_retention=FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time()) + 100), ) with pytest.raises(AccessDenied): self.api.delete_file_version(created_file.id_, 'file') self.api.delete_file_version(created_file.id_, 'file', bypass_governance=True) with pytest.raises(FileNotPresent): bucket.get_file_info_by_name(created_file.file_name) b2-sdk-python-2.8.0/test/unit/b2http/000077500000000000000000000000001474454370000172735ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/b2http/__init__.py000066400000000000000000000005121474454370000214020ustar00rootroot00000000000000###################################################################### # # File: test/unit/b2http/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/b2http/test_b2http.py000066400000000000000000000370131474454370000221130ustar00rootroot00000000000000###################################################################### # # File: test/unit/b2http/test_b2http.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import datetime import locale import sys from unittest.mock import MagicMock, call, patch import apiver_deps import pytest import requests from apiver_deps import USER_AGENT, B2Http, B2HttpApiConfig, ClockSkewHook from apiver_deps_exception import ( B2ConnectionError, BadDateFormat, BadJson, BadRequest, BrokenPipe, ClockSkew, ConnectionReset, PotentialS3EndpointPassedAsRealm, ServiceError, TooManyRequests, UnknownError, UnknownHost, ) from b2sdk._internal.b2http import setlocale from ..test_base import TestBase class TestTranslateErrors: def test_ok(self): response = MagicMock() response.status_code = 200 actual = B2Http._translate_errors(lambda: response) assert response == actual def test_partial_content(self): response = MagicMock() response.status_code = 206 actual = B2Http._translate_errors(lambda: response) assert response == actual def test_b2_error(self): response = MagicMock() response.status_code = 503 response.content = b'{"status": 503, "code": "server_busy", "message": "busy"}' with pytest.raises(ServiceError): B2Http._translate_errors(lambda: response) def test_broken_pipe(self): def fcn(): raise requests.ConnectionError( requests.packages.urllib3.exceptions.ProtocolError( 'dummy', OSError(20, 'Broken pipe') ) ) with pytest.raises(BrokenPipe): B2Http._translate_errors(fcn) def test_unknown_host(self): def fcn(): raise requests.ConnectionError( requests.packages.urllib3.exceptions.MaxRetryError( 'AAA nodename nor servname provided, or not known AAA', 'http://example.com' ) ) with pytest.raises(UnknownHost): B2Http._translate_errors(fcn) def test_connection_error(self): def fcn(): raise requests.ConnectionError('a message') with pytest.raises(B2ConnectionError): B2Http._translate_errors(fcn) def test_connection_reset(self): class SysCallError(Exception): pass def fcn(): raise SysCallError('(104, ECONNRESET)') with pytest.raises(ConnectionReset): B2Http._translate_errors(fcn) def test_unknown_error(self): def fcn(): raise Exception('a message') with pytest.raises(UnknownError): B2Http._translate_errors(fcn) def test_too_many_requests(self): response = MagicMock() response.status_code = 429 response.headers = {'retry-after': 1} response.content = ( b'{"status": 429, "code": "Too Many requests", "message": "retry after some time"}' ) with pytest.raises(TooManyRequests): B2Http._translate_errors(lambda: response) def test_invalid_json(self): response = MagicMock() response.status_code = 400 response.content = b'{' * 500 response.url = 'https://example.com' with pytest.raises(BadRequest) as exc_info: B2Http._translate_errors(lambda: response) assert str(exc_info.value) == f'{response.content.decode()} (non_json_response)' def test_potential_s3_endpoint_passed_as_realm(self): response = MagicMock() response.status_code = 400 response.content = b'' response.url = 'https://s3.us-west-000.backblazeb2.com' with pytest.raises(PotentialS3EndpointPassedAsRealm): B2Http._translate_errors(lambda: response) @pytest.mark.apiver(to_ver=2) def test_bucket_id_not_found(self): from b2sdk.v2.exception import BucketIdNotFound, v3BucketIdNotFound def fcn(): raise v3BucketIdNotFound('bucket_id') with pytest.raises(BucketIdNotFound) as exc_info: B2Http._translate_errors(fcn) assert str(exc_info.value) == 'Bucket with id=bucket_id not found (bad_bucket_id)' def test_b2_error__nginx_html(): """ While errors with HTML description should not happen, we should not crash on them. """ response = MagicMock() response.status_code = 502 response.content = b'

502 Bad Gateway

' with pytest.raises(ServiceError) as exc_info: B2Http._translate_errors(lambda: response) assert response.content.decode('utf-8') in str(exc_info.value) def test_b2_error__invalid_error_format(): """ Handling of invalid error format. If server returns valid JSON, but not matching B2 error schema, we should still raise ServiceError. """ response = MagicMock() response.status_code = 503 # valid JSON, but not a valid B2 error (it should be a dict, not a list) response.content = b'[]' with pytest.raises(ServiceError) as exc_info: B2Http._translate_errors(lambda: response) assert '503' in str(exc_info.value) def test_b2_error__invalid_error_values(): """ Handling of invalid error values. If server returns valid JSON, but not matching B2 error schema, we should still raise ServiceError. """ response = MagicMock() response.status_code = 503 # valid JSON, but not a valid B2 error (code and status values (and therefore types!) are swapped) response.content = b'{"code": 503, "message": "Service temporarily unavailable", "status": "service_unavailable"}' with pytest.raises(ServiceError) as exc_info: B2Http._translate_errors(lambda: response) assert '503 Service temporarily unavailable' in str(exc_info.value) class TestTranslateAndRetry(TestBase): def setUp(self): self.response = MagicMock() self.response.status_code = 200 def test_works_first_try(self): fcn = MagicMock() fcn.side_effect = [self.response] self.assertIs(self.response, B2Http._translate_and_retry(fcn, 3)) def test_non_retryable(self): with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [BadJson('a'), self.response] with self.assertRaises(BadJson): B2Http._translate_and_retry(fcn, 3) self.assertEqual([], mock_time.mock_calls) def test_works_second_try(self): with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [ServiceError('a'), self.response] self.assertIs(self.response, B2Http._translate_and_retry(fcn, 3)) self.assertEqual([call(1.0)], mock_time.mock_calls) def test_never_works(self): with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [ ServiceError('a'), ServiceError('a'), ServiceError('a'), self.response, ] with self.assertRaises(ServiceError): B2Http._translate_and_retry(fcn, 3) self.assertEqual([call(1.0), call(1.5)], mock_time.mock_calls) def test_too_many_requests_works_after_sleep(self): with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [TooManyRequests(retry_after_seconds=2), self.response] self.assertIs(self.response, B2Http._translate_and_retry(fcn, 3)) self.assertEqual([call(2)], mock_time.mock_calls) def test_too_many_requests_failed_after_sleep(self): with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [ TooManyRequests(retry_after_seconds=2), TooManyRequests(retry_after_seconds=5), ] with self.assertRaises(TooManyRequests): B2Http._translate_and_retry(fcn, 2) self.assertEqual([call(2)], mock_time.mock_calls) def test_too_many_requests_retry_header_combination_one(self): # If the first response didn't have a header, second one has, and third one doesn't have, what should happen? with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [ TooManyRequests(retry_after_seconds=2), TooManyRequests(), TooManyRequests(retry_after_seconds=2), self.response, ] self.assertIs(self.response, B2Http._translate_and_retry(fcn, 4)) self.assertEqual([call(2), call(1.5), call(2)], mock_time.mock_calls) def test_too_many_requests_retry_header_combination_two(self): # If the first response had header, and the second did not, but the third has header again, what should happen? with patch('time.sleep') as mock_time: fcn = MagicMock() fcn.side_effect = [ TooManyRequests(), TooManyRequests(retry_after_seconds=5), TooManyRequests(), self.response, ] self.assertIs(self.response, B2Http._translate_and_retry(fcn, 4)) self.assertEqual([call(1.0), call(5), call(2.25)], mock_time.mock_calls) class TestB2Http(TestBase): URL = 'http://example.com' UA_APPEND = None HEADERS = dict(my_header='my_value') EXPECTED_HEADERS = {'my_header': 'my_value', 'User-Agent': USER_AGENT} EXPECTED_JSON_HEADERS = { **EXPECTED_HEADERS, 'Content-Type': 'application/json', 'Accept': 'application/json', } PARAMS = dict(fileSize=100) PARAMS_JSON_BYTES = b'{"fileSize": 100}' def setUp(self): self.session = MagicMock() self.response = MagicMock() requests = MagicMock() requests.Session.return_value = self.session if apiver_deps.V <= 1: self.b2_http = B2Http( requests, install_clock_skew_hook=False, user_agent_append=self.UA_APPEND ) else: self.b2_http = B2Http( B2HttpApiConfig( requests.Session, install_clock_skew_hook=False, user_agent_append=self.UA_APPEND, ) ) def test_post_json_return_json(self): self.session.request.return_value = self.response self.response.status_code = 200 self.response.content = b'{"color": "blue"}' response_dict = self.b2_http.post_json_return_json(self.URL, self.HEADERS, self.PARAMS) self.assertEqual({'color': 'blue'}, response_dict) (pos_args, kw_args) = self.session.request.call_args assert pos_args[:2] == ('POST', self.URL) assert kw_args['headers'] == self.EXPECTED_JSON_HEADERS assert kw_args['data'] == self.PARAMS_JSON_BYTES def test_callback(self): callback = MagicMock() callback.pre_request = MagicMock() callback.post_request = MagicMock() self.b2_http.add_callback(callback) self.session.request.return_value = self.response self.response.status_code = 200 self.response.content = b'{"color": "blue"}' self.b2_http.post_json_return_json(self.URL, self.HEADERS, self.PARAMS) callback.pre_request.assert_called_with( 'POST', 'http://example.com', self.EXPECTED_JSON_HEADERS ) callback.post_request.assert_called_with( 'POST', 'http://example.com', self.EXPECTED_JSON_HEADERS, self.response ) def test_get_content(self): self.session.request.return_value = self.response self.response.status_code = 200 with self.b2_http.get_content(self.URL, self.HEADERS) as r: self.assertIs(self.response, r) self.session.request.assert_called_with( 'GET', self.URL, headers=self.EXPECTED_HEADERS, data=None, params=None, stream=True, timeout=(B2Http.CONNECTION_TIMEOUT, B2Http.TIMEOUT), ) self.response.close.assert_not_called() # prevent premature close() on requests.Response def test_head_content(self): self.session.request.return_value = self.response self.response.status_code = 200 self.response.headers = {'color': 'blue'} response = self.b2_http.head_content(self.URL, self.HEADERS) self.assertEqual({'color': 'blue'}, response.headers) (pos_args, kw_args) = self.session.request.call_args assert pos_args[:2] == ('HEAD', self.URL) assert kw_args['headers'] == self.EXPECTED_HEADERS class TestB2HttpUserAgentAppend(TestB2Http): UA_APPEND = 'ua_extra_string' EXPECTED_HEADERS = {**TestB2Http.EXPECTED_HEADERS, 'User-Agent': f'{USER_AGENT} {UA_APPEND}'} EXPECTED_JSON_HEADERS = { **TestB2Http.EXPECTED_JSON_HEADERS, 'User-Agent': EXPECTED_HEADERS['User-Agent'], } class TestSetLocaleContextManager(TestBase): def test_set_locale_context_manager(self): # C.UTF-8 on Ubuntu 18.04 Bionic, C.utf8 on Ubuntu 22.04 Jammy # Neither macOS nor Windows have C.UTF-8 locale, and they use `en_US.UTF-8`. # Since Python 3.12, locale.normalize no longer falls back # to the `en_US` version, so we're providing it here manually. test_locale = locale.normalize('C.UTF-8' if sys.platform == 'linux' else 'en_US.UTF-8') other_locale = 'C' saved = locale.setlocale(locale.LC_ALL) if saved == test_locale: test_locale, other_locale = other_locale, test_locale locale.setlocale(locale.LC_ALL, other_locale) with setlocale(test_locale): assert locale.setlocale(category=locale.LC_ALL) == test_locale locale.setlocale(locale.LC_ALL, saved) class TestClockSkewHook(TestBase): def test_bad_format(self): response = MagicMock() response.headers = {'Date': 'bad format'} with self.assertRaises(BadDateFormat): ClockSkewHook().post_request('POST', 'http://example.com', {}, response) def test_bad_month(self): response = MagicMock() response.headers = {'Date': 'Fri, 16 XXX 2016 20:52:30 GMT'} with self.assertRaises(BadDateFormat): ClockSkewHook().post_request('POST', 'http://example.com', {}, response) def test_no_skew(self): now = datetime.datetime.now(datetime.timezone.utc) now_str = now.strftime('%a, %d %b %Y %H:%M:%S GMT') response = MagicMock() response.headers = {'Date': now_str} ClockSkewHook().post_request('POST', 'http://example.com', {}, response) def test_positive_skew(self): now = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=11) now_str = now.strftime('%a, %d %b %Y %H:%M:%S GMT') response = MagicMock() response.headers = {'Date': now_str} with self.assertRaises(ClockSkew): ClockSkewHook().post_request('POST', 'http://example.com', {}, response) def test_negative_skew(self): now = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=-11) now_str = now.strftime('%a, %d %b %Y %H:%M:%S GMT') response = MagicMock() response.headers = {'Date': now_str} with self.assertRaises(ClockSkew): ClockSkewHook().post_request('POST', 'http://example.com', {}, response) b2-sdk-python-2.8.0/test/unit/bucket/000077500000000000000000000000001474454370000173455ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/bucket/__init__.py000066400000000000000000000005121474454370000214540ustar00rootroot00000000000000###################################################################### # # File: test/unit/bucket/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/bucket/test_bucket.py000066400000000000000000004020351474454370000222370ustar00rootroot00000000000000###################################################################### # # File: test/unit/bucket/test_bucket.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import contextlib import dataclasses import datetime import io import os import pathlib import platform import tempfile import time import unittest.mock as mock from contextlib import suppress from io import BytesIO import apiver_deps import pytest from apiver_deps_exception import ( AccessDenied, AlreadyFailed, B2ConnectionError, B2Error, B2RequestTimeoutDuringUpload, BucketIdNotFound, DestinationDirectoryDoesntAllowOperation, DestinationDirectoryDoesntExist, DestinationIsADirectory, DestinationParentIsNotADirectory, DisablingFileLockNotSupported, FileSha1Mismatch, InvalidAuthToken, InvalidMetadataDirective, InvalidRange, InvalidUploadSource, MaxRetriesExceeded, RestrictedBucketMissing, SourceReplicationConflict, SSECKeyError, UnsatisfiableRange, ) from test.helpers import NonSeekableIO, assert_dict_equal_ignore_extra from ..test_base import TestBase, create_key if apiver_deps.V <= 1: from apiver_deps import DownloadDestBytes, PreSeekedDownloadDest from apiver_deps import FileVersionInfo as VFileVersionInfo else: DownloadDestBytes, PreSeekedDownloadDest = ( None, None, ) # these classes are not present, thus not needed, in v2 from apiver_deps import FileVersion as VFileVersionInfo from apiver_deps import ( LARGE_FILE_SHA1, NO_RETENTION_FILE_SETTING, SSE_B2_AES, SSE_NONE, AbstractDownloader, AbstractProgressListener, B2Api, B2HttpApiConfig, B2Session, Bucket, BucketFactory, BucketRetentionSetting, BucketSimulator, CopySource, DownloadedFile, DownloadVersion, DummyCache, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, FakeResponse, FileRetentionSetting, FileSimulator, Filter, InMemoryCache, LargeFileUploadState, LegalHold, MetadataDirectiveMode, ParallelDownloader, Part, Range, RawSimulator, ReplicationConfiguration, ReplicationRule, RetentionMode, RetentionPeriod, SimpleDownloader, StubAccountInfo, UploadMode, UploadSourceBytes, UploadSourceLocalFile, WriteIntent, hex_sha1_of_bytes, ) pytestmark = [pytest.mark.apiver(from_ver=1)] SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_key', key_id='some-id'), ) SSE_C_AES_NO_SECRET = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=None, key_id='some-id'), ) SSE_C_AES_2 = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_other_key', key_id='some-id-2'), ) SSE_C_AES_2_NO_SECRET = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=None, key_id='some-id-2'), ) SSE_C_AES_FROM_SERVER = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(key_id=None, secret=None), ) REPLICATION = ReplicationConfiguration( rules=[ ReplicationRule( destination_bucket_id='c5f35d53a90a7ea284fb0719', name='replication-us-west', ), ReplicationRule( destination_bucket_id='55f34d53a96a7ea284fb0719', name='replication-us-west-2', file_name_prefix='replica/', is_enabled=False, priority=255, ), ], source_key_id='10053d55ae26b790000000006', source_to_destination_key_mapping={ '10053d55ae26b790000000045': '10053d55ae26b790000000004', '10053d55ae26b790000000046': '10053d55ae26b790030000004', }, ) def write_file(path, data): with open(path, 'wb') as f: f.write(data) class StubProgressListener(AbstractProgressListener): """ Implementation of a progress listener that remembers what calls were made, and returns them as a short string to use in unit tests. For a total byte count of 100, and updates at 33 and 66, the returned string looks like: "100: 33 66" """ def __init__(self): self.total = None self.history = [] self.last_byte_count = 0 def get_history(self): return ' '.join(self.history) def set_total_bytes(self, total_byte_count): assert total_byte_count is not None assert self.total is None, 'set_total_bytes called twice' self.total = total_byte_count assert len(self.history) == 0, self.history self.history.append('%d:' % (total_byte_count,)) def bytes_completed(self, byte_count): self.last_byte_count = byte_count self.history.append(str(byte_count)) def is_valid(self, **kwargs): valid, _ = self.is_valid_reason(**kwargs) return valid def is_valid_reason(self, check_progress=True, check_monotonic_progress=False): progress_end = -1 if self.history[progress_end] == 'closed': progress_end = -2 # self.total != self.last_byte_count may be a consequence of non-monotonic # progress, so we want to check this first if check_monotonic_progress: prev = 0 for val in map(int, self.history[1:progress_end]): if val < prev: return False, 'non-monotonic progress' prev = val if self.total != self.last_byte_count: return False, 'total different than last_byte_count' if check_progress and len(self.history[1:progress_end]) < 2: return False, 'progress in history has less than 2 entries' return True, '' def close(self): self.history.append('closed') class CanRetry(B2Error): """ An exception that can be retryable, or not. """ def __init__(self, can_retry): super().__init__(None, None, None, None, None) self.can_retry = can_retry def should_retry_upload(self): return self.can_retry def bucket_ls(bucket, *args, show_versions=False, **kwargs): if apiver_deps.V <= 1: ls_all_versions_kwarg = {'show_versions': show_versions} else: ls_all_versions_kwarg = {'latest_only': not show_versions} return bucket.ls(*args, **ls_all_versions_kwarg, **kwargs) @pytest.fixture def exact_filename_match_ls_setup(bucket): data = b'hello world' filename1 = 'hello.txt' hidden_file = filename1 + 'postfix' filename3 = filename1 + 'postfix3' files = [ bucket.upload_bytes(data, filename1), bucket.upload_bytes(data, hidden_file), bucket.upload_bytes(data, filename3), ] bucket.hide_file(hidden_file) return files @pytest.mark.apiver(from_ver=2, to_ver=2) def test_bucket_ls__pre_v3_does_not_match_exact_filename(bucket, exact_filename_match_ls_setup): assert not list(bucket.ls(exact_filename_match_ls_setup[0].file_name)) @pytest.mark.apiver(from_ver=2) def test_bucket_ls__matches_exact_filename(bucket, exact_filename_match_ls_setup, apiver_int): assert len(list(bucket.ls())) == 2 assert len(list(bucket.ls(latest_only=False))) == 4 kwargs = {} if apiver_int < 3: kwargs['folder_to_list_can_be_a_file'] = True assert [ fv.file_name for fv, _ in bucket.ls(exact_filename_match_ls_setup[0].file_name, **kwargs) ] == ['hello.txt'] # hidden file should not be returned unless latest_only is False assert len(list(bucket.ls(exact_filename_match_ls_setup[1].file_name, **kwargs))) == 0 assert ( len( list(bucket.ls(exact_filename_match_ls_setup[1].file_name, **kwargs, latest_only=False)) ) == 2 ) @pytest.mark.apiver(from_ver=2) def test_bucket_ls__matches_exact_filename__wildcard( bucket, exact_filename_match_ls_setup, apiver_int ): kwargs = {'with_wildcard': True, 'recursive': True} if apiver_int < 3: kwargs['folder_to_list_can_be_a_file'] = True assert [ fv.file_name for fv, _ in bucket.ls(exact_filename_match_ls_setup[0].file_name, **kwargs) ] == ['hello.txt'] # hidden file should not be returned unless latest_only is False assert len(list(bucket.ls(exact_filename_match_ls_setup[1].file_name, **kwargs))) == 0 assert ( len( list(bucket.ls(exact_filename_match_ls_setup[1].file_name, **kwargs, latest_only=False)) ) == 2 ) class TestCaseWithBucket(TestBase): RAW_SIMULATOR_CLASS = RawSimulator CACHE_CLASS = DummyCache def get_api(self): return B2Api( self.account_info, cache=self.CACHE_CLASS(), api_config=B2HttpApiConfig(_raw_api_class=self.RAW_SIMULATOR_CLASS), ) def setUp(self): self.bucket_name = 'my-bucket' self.account_info = StubAccountInfo() self.api = self.get_api() self.simulator = self.api.session.raw_api (self.account_id, self.master_key) = self.simulator.create_account() self.api.authorize_account( application_key_id=self.account_id, application_key=self.master_key, realm='production', ) self.api_url = self.account_info.get_api_url() self.account_auth_token = self.account_info.get_account_auth_token() self.bucket = self.api.create_bucket(self.bucket_name, 'allPublic') self.bucket_id = self.bucket.id_ def bucket_ls(self, *args, show_versions=False, **kwargs): return bucket_ls(self.bucket, *args, show_versions=show_versions, **kwargs) def assertBucketContents(self, expected, *args, **kwargs): """ *args and **kwargs are passed to self.bucket_ls() """ actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls(*args, **kwargs) ] self.assertEqual(expected, actual) def _make_data(self, approximate_length): """ Generate a sequence of bytes to use in testing an upload. Don't repeat a short pattern, so we're sure that the different parts of a large file are actually different. Returns bytes. """ fragments = [] so_far = 0 while so_far < approximate_length: fragment = ('%d:' % so_far).encode('utf-8') so_far += len(fragment) fragments.append(fragment) return b''.join(fragments) def _check_file_contents(self, file_name, expected_contents): contents = self._download_file(file_name) self.assertEqual(expected_contents, contents) def _check_large_file_sha1(self, file_name, expected_sha1): file_info = self.bucket.get_file_info_by_name(file_name).file_info if expected_sha1: assert LARGE_FILE_SHA1 in file_info assert file_info[LARGE_FILE_SHA1] == expected_sha1 else: assert LARGE_FILE_SHA1 not in file_info def _download_file(self, file_name): with FileSimulator.dont_check_encryption(): if apiver_deps.V <= 1: download = DownloadDestBytes() self.bucket.download_file_by_name(file_name, download) return download.get_bytes_written() else: with io.BytesIO() as bytes_io: downloaded_file = self.bucket.download_file_by_name(file_name) downloaded_file.save(bytes_io) return bytes_io.getvalue() class TestReauthorization(TestCaseWithBucket): def testCreateBucket(self): class InvalidAuthTokenWrapper: def __init__(self, original_function): self.__original_function = original_function self.__name__ = original_function.__name__ self.__called = False def __call__(self, *args, **kwargs): if self.__called: return self.__original_function(*args, **kwargs) self.__called = True raise InvalidAuthToken('message', 401) self.simulator.create_bucket = InvalidAuthTokenWrapper(self.simulator.create_bucket) self.bucket = self.api.create_bucket('your-bucket', 'allPublic') class TestListParts(TestCaseWithBucket): @pytest.mark.apiver(to_ver=1) def testEmpty(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) self.assertEqual([], list(self.bucket.list_parts(file1.file_id, batch_size=1))) @pytest.mark.apiver(to_ver=1) def testThree(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) content = b'hello world' content_sha1 = hex_sha1_of_bytes(content) large_file_upload_state = mock.MagicMock() large_file_upload_state.has_error.return_value = False self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 1, large_file_upload_state ).result() self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 2, large_file_upload_state ).result() self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 3, large_file_upload_state ).result() expected_parts = [ Part('9999', 1, 11, content_sha1), Part('9999', 2, 11, content_sha1), Part('9999', 3, 11, content_sha1), ] self.assertEqual(expected_parts, list(self.bucket.list_parts(file1.file_id, batch_size=1))) class TestUploadPart(TestCaseWithBucket): @pytest.mark.apiver(to_ver=1) def test_error_in_state(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) content = b'hello world' file_progress_listener = mock.MagicMock() large_file_upload_state = LargeFileUploadState(file_progress_listener) large_file_upload_state.set_error('test error') try: self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 1, large_file_upload_state, ).result() self.fail('should have thrown') except AlreadyFailed: pass class TestListUnfinished(TestCaseWithBucket): def test_empty(self): self.assertEqual([], list(self.bucket.list_unfinished_large_files())) @pytest.mark.apiver(to_ver=1) def test_one(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) self.assertEqual([file1], list(self.bucket.list_unfinished_large_files())) @pytest.mark.apiver(to_ver=1) def test_three(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) file2 = self.bucket.start_large_file('file2.txt', 'text/plain', {}) file3 = self.bucket.start_large_file('file3.txt', 'text/plain', {}) self.assertEqual( [file1, file2, file3], list(self.bucket.list_unfinished_large_files(batch_size=1)) ) @pytest.mark.apiver(to_ver=1) def test_prefix(self): self.bucket.start_large_file('fileA', 'text/plain', {}) file2 = self.bucket.start_large_file('fileAB', 'text/plain', {}) file3 = self.bucket.start_large_file('fileABC', 'text/plain', {}) self.assertEqual( [file2, file3], list( self.bucket.list_unfinished_large_files( batch_size=1, prefix='fileAB', ), ), ) def _make_file(self, file_id, file_name): return self.bucket.start_large_file(file_name, 'text/plain', {}) class TestGetFileInfo(TestCaseWithBucket): def test_version_by_name(self): data = b'hello world' a_id = self.bucket.upload_bytes(data, 'a').id_ info = self.bucket.get_file_info_by_name('a') if apiver_deps.V <= 1: self.assertIsInstance(info, VFileVersionInfo) else: self.assertIsInstance(info, DownloadVersion) expected = (a_id, 'a', 11, 'b2/x-auto', 'none', NO_RETENTION_FILE_SETTING, LegalHold.UNSET) actual = ( info.id_, info.file_name, info.size, info.content_type, info.server_side_encryption.mode.value, info.file_retention, info.legal_hold, ) self.assertEqual(expected, actual) def test_version_by_name_file_lock(self): bucket = self.api.create_bucket( 'my-bucket-with-file-lock', 'allPublic', is_file_lock_enabled=True ) data = b'hello world' legal_hold = LegalHold.ON file_retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 100) bucket.upload_bytes(data, 'a', file_retention=file_retention, legal_hold=legal_hold) file_version = bucket.get_file_info_by_name('a') actual = (file_version.legal_hold, file_version.file_retention) self.assertEqual((legal_hold, file_retention), actual) low_perm_account_info = StubAccountInfo() low_perm_api = B2Api(low_perm_account_info) low_perm_api.session.raw_api = self.simulator low_perm_key = create_key( self.api, key_name='lowperm', capabilities=[ 'listKeys', 'listBuckets', 'listFiles', 'readFiles', ], ) low_perm_api.authorize_account( application_key_id=low_perm_key.id_, application_key=low_perm_key.application_key, realm='production', ) low_perm_bucket = low_perm_api.get_bucket_by_name('my-bucket-with-file-lock') file_version = low_perm_bucket.get_file_info_by_name('a') actual = (file_version.legal_hold, file_version.file_retention) expected = (LegalHold.UNKNOWN, FileRetentionSetting(RetentionMode.UNKNOWN)) self.assertEqual(expected, actual) def test_version_by_id(self): data = b'hello world' b_id = self.bucket.upload_bytes(data, 'b').id_ info = self.bucket.get_file_info_by_id(b_id) self.assertIsInstance(info, VFileVersionInfo) expected = (b_id, 'b', 11, 'upload', 'b2/x-auto', 'none') actual = ( info.id_, info.file_name, info.size, info.action, info.content_type, info.server_side_encryption.mode.value, ) self.assertEqual(expected, actual) class TestLs(TestCaseWithBucket): def test_empty(self): self.assertEqual([], list(self.bucket_ls('foo'))) def test_one_file_at_root(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '') def test_three_files_at_root(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb') self.bucket.upload_bytes(data, 'ccc') expected = [ ('a', 11, 'upload', None), ('bb', 11, 'upload', None), ('ccc', 11, 'upload', None), ] self.assertBucketContents(expected, '') def test_three_files_in_dir(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb/1') self.bucket.upload_bytes(data, 'bb/2/sub1') self.bucket.upload_bytes(data, 'bb/2/sub2') self.bucket.upload_bytes(data, 'bb/3') self.bucket.upload_bytes(data, 'ccc') expected = [ ('bb/1', 11, 'upload', None), ('bb/2/sub1', 11, 'upload', 'bb/2/'), ('bb/3', 11, 'upload', None), ] self.assertBucketContents(expected, 'bb', fetch_count=1) def test_three_files_multiple_versions(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb/1') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/3') self.bucket.upload_bytes(data, 'ccc') expected = [ ('9998', 'bb/1', 11, 'upload', None), ('9995', 'bb/2', 11, 'upload', None), ('9996', 'bb/2', 11, 'upload', None), ('9997', 'bb/2', 11, 'upload', None), ('9994', 'bb/3', 11, 'upload', None), ] actual = [ (info.id_, info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('bb', show_versions=True, fetch_count=1) ] self.assertEqual(expected, actual) @pytest.mark.apiver(to_ver=1) def test_started_large_file(self): self.bucket.start_large_file('hello.txt') expected = [('hello.txt', 0, 'start', None)] self.assertBucketContents(expected, '', show_versions=True) def test_hidden_file(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') self.bucket.hide_file('hello.txt') expected = [('hello.txt', 0, 'hide', None), ('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_unhidden_file(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') self.bucket.hide_file('hello.txt') self.bucket.unhide_file('hello.txt') expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_delete_file_version(self): data = b'hello world' file_id = self.bucket.upload_bytes(data, 'hello.txt').id_ data = b'hello new world' self.bucket.upload_bytes(data, 'hello.txt') self.bucket.delete_file_version(file_id, 'hello.txt') expected = [('hello.txt', 15, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_delete_file_version_bypass_governance(self): data = b'hello world' file_id = self.bucket.upload_bytes( data, 'hello.txt', file_retention=FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time()) + 100), ).id_ with pytest.raises(AccessDenied): self.bucket.delete_file_version(file_id, 'hello.txt') self.bucket.delete_file_version(file_id, 'hello.txt', bypass_governance=True) self.assertBucketContents([], '', show_versions=True) def test_non_recursive_returns_folder_names(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/1/test-1.txt') self.bucket.upload_bytes(data, 'b/2/test-2.txt') self.bucket.upload_bytes(data, 'b/3/test-3.txt') self.bucket.upload_bytes(data, 'b/3/test-4.txt') # Since inside `b` there are 3 directories, we get three results, # with a first file for each of them. expected = [ ('b/1/test-1.txt', len(data), 'upload', 'b/1/'), ('b/2/test-2.txt', len(data), 'upload', 'b/2/'), ('b/3/test-3.txt', len(data), 'upload', 'b/3/'), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('b/') ] self.assertEqual(expected, actual) def test_recursive_returns_no_folder_names(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/1/test-1.txt') self.bucket.upload_bytes(data, 'b/2/test-2.txt') self.bucket.upload_bytes(data, 'b/3/test-3.txt') self.bucket.upload_bytes(data, 'b/3/test-4.txt') expected = [ ('b/1/test-1.txt', len(data), 'upload', None), ('b/2/test-2.txt', len(data), 'upload', None), ('b/3/test-3.txt', len(data), 'upload', None), ('b/3/test-4.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('b/', recursive=True) ] self.assertEqual(expected, actual) def test_wildcard_matching(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/1/test-1.txt') self.bucket.upload_bytes(data, 'b/2/test-2.csv') self.bucket.upload_bytes(data, 'b/2/test-3.txt') self.bucket.upload_bytes(data, 'b/3/test-4.jpg') self.bucket.upload_bytes(data, 'b/3/test-4.txt') self.bucket.upload_bytes(data, 'b/3/test-5.txt') expected = [ ('b/1/test-1.txt', len(data), 'upload', None), ('b/2/test-3.txt', len(data), 'upload', None), ('b/3/test-4.txt', len(data), 'upload', None), ('b/3/test-5.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('b/*.txt', recursive=True, with_wildcard=True) ] self.assertEqual(expected, actual) def test_wildcard_matching_including_root(self): data = b'hello world' self.bucket.upload_bytes(data, 'b/1/test.txt') self.bucket.upload_bytes(data, 'b/2/test.txt') self.bucket.upload_bytes(data, 'b/3/test.txt') self.bucket.upload_bytes(data, 'test.txt') expected = [ ('b/1/test.txt', len(data), 'upload', None), ('b/2/test.txt', len(data), 'upload', None), ('b/3/test.txt', len(data), 'upload', None), ('test.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('*.txt', recursive=True, with_wildcard=True) ] self.assertEqual(expected, actual) def test_wildcard_matching_directory(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/2/test.txt') self.bucket.upload_bytes(data, 'b/3/test.jpg') self.bucket.upload_bytes(data, 'b/3/test.txt') self.bucket.upload_bytes(data, 'c/4/test.txt') expected = [ ('b/2/test.txt', len(data), 'upload', None), ('b/3/test.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('b/*/test.txt', recursive=True, with_wildcard=True) ] self.assertEqual(expected, actual) def test_single_character_matching(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/2/test.csv') self.bucket.upload_bytes(data, 'b/2/test.txt') self.bucket.upload_bytes(data, 'b/2/test.tsv') expected = [ ('b/2/test.csv', len(data), 'upload', None), ('b/2/test.tsv', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('b/2/test.?sv', recursive=True, with_wildcard=True) ] self.assertEqual(expected, actual) def test_sequence_matching(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/2/test.csv') self.bucket.upload_bytes(data, 'b/2/test.ksv') self.bucket.upload_bytes(data, 'b/2/test.tsv') expected = [ ('b/2/test.csv', len(data), 'upload', None), ('b/2/test.tsv', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( 'b/2/test.[tc]sv', recursive=True, with_wildcard=True ) ] self.assertEqual(expected, actual) def test_negative_sequence_matching(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/2/test.csv') self.bucket.upload_bytes(data, 'b/2/test.ksv') self.bucket.upload_bytes(data, 'b/2/test.tsv') expected = [ ('b/2/test.tsv', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( 'b/2/test.[!ck]sv', recursive=True, with_wildcard=True ) ] self.assertEqual(expected, actual) def test_matching_wildcard_named_file(self): data = b'hello world' self.bucket.upload_bytes(data, 'a/*.txt') self.bucket.upload_bytes(data, 'a/1.txt') self.bucket.upload_bytes(data, 'a/2.txt') expected = [ ('a/*.txt', len(data), 'upload', None), ('a/1.txt', len(data), 'upload', None), ('a/2.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('a/*.txt', recursive=True, with_wildcard=True) ] self.assertEqual(expected, actual) def test_matching_single_question_mark_named_file(self): data = b'hello world' self.bucket.upload_bytes(data, 'b/?.txt') self.bucket.upload_bytes(data, 'b/a.txt') self.bucket.upload_bytes(data, 'b/b.txt') expected = [ ('b/?.txt', len(data), 'upload', None), ('b/a.txt', len(data), 'upload', None), ('b/b.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('b/?.txt', recursive=True, with_wildcard=True) ] self.assertEqual(expected, actual) def test_wildcard_requires_recursive(self): with pytest.raises(ValueError): # Since ls is a generator, we need to actually fetch something from it. next(self.bucket_ls('*.txt', recursive=False, with_wildcard=True)) def test_matching_exact_filename(self): data = b'hello world' self.bucket.upload_bytes(data, 'b/a.txt') self.bucket.upload_bytes(data, 'b/b.txt') expected = [ ('b/a.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls('b/a.txt', recursive=True, with_wildcard=True) ] self.assertEqual(expected, actual) def test_filters_wildcard_matching(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/1/test-1.txt') self.bucket.upload_bytes(data, 'b/2/test-2.csv') self.bucket.upload_bytes(data, 'b/2/test-3.txt') self.bucket.upload_bytes(data, 'b/3/test-4.jpg') self.bucket.upload_bytes(data, 'b/3/test-4.txt') self.bucket.upload_bytes(data, 'b/3/test-5.txt') expected = [ ('b/1/test-1.txt', len(data), 'upload', None), ('b/2/test-3.txt', len(data), 'upload', None), ('b/3/test-4.txt', len(data), 'upload', None), ('b/3/test-5.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( 'b/', recursive=True, filters=[Filter.include('*.txt')], ) ] self.assertEqual(expected, actual) def test_filters_wildcard_matching_including_root(self): data = b'hello world' self.bucket.upload_bytes(data, 'b/1/test.csv') self.bucket.upload_bytes(data, 'b/1/test.txt') self.bucket.upload_bytes(data, 'b/2/test.tsv') self.bucket.upload_bytes(data, 'b/2/test.txt') self.bucket.upload_bytes(data, 'b/3/test.txt') self.bucket.upload_bytes(data, 'test.txt') self.bucket.upload_bytes(data, 'test.csv') expected = [ ('b/1/test.txt', len(data), 'upload', None), ('b/2/test.txt', len(data), 'upload', None), ('b/3/test.txt', len(data), 'upload', None), ('test.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls(recursive=True, filters=[Filter.include('*.txt')]) ] self.assertEqual(expected, actual) expected = [ ('b/1/test.csv', len(data), 'upload', None), ('b/2/test.tsv', len(data), 'upload', None), ('test.csv', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls(recursive=True, filters=[Filter.exclude('*.txt')]) ] self.assertEqual(expected, actual) def test_filters_single_character_matching(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/2/test.csv') self.bucket.upload_bytes(data, 'b/2/test.txt') self.bucket.upload_bytes(data, 'b/2/test.tsv') expected = [ ('b/2/test.csv', len(data), 'upload', None), ('b/2/test.tsv', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( recursive=True, filters=[Filter.include('b/2/test.?sv')], ) ] self.assertEqual(expected, actual) expected = [ ('a', len(data), 'upload', None), ('b/2/test.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( recursive=True, filters=[Filter.exclude('b/2/test.?sv')], ) ] self.assertEqual(expected, actual) def test_filters_sequence_matching(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/2/test.csv') self.bucket.upload_bytes(data, 'b/2/test.ksv') self.bucket.upload_bytes(data, 'b/2/test.tsv') expected = [ ('b/2/test.csv', len(data), 'upload', None), ('b/2/test.tsv', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( recursive=True, filters=[Filter.include('b/2/test.[tc]sv')], ) ] self.assertEqual(expected, actual) expected = [ ('a', len(data), 'upload', None), ('b/2/test.ksv', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( recursive=True, filters=[Filter.exclude('b/2/test.[tc]sv')], ) ] self.assertEqual(expected, actual) def test_filters_negative_sequence_matching(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'b/2/test.csv') self.bucket.upload_bytes(data, 'b/2/test.ksv') self.bucket.upload_bytes(data, 'b/2/test.tsv') expected = [ ('b/2/test.tsv', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( recursive=True, filters=[Filter.include('b/2/test.[!ck]sv')], ) ] self.assertEqual(expected, actual) expected = [ ('a', len(data), 'upload', None), ('b/2/test.csv', len(data), 'upload', None), ('b/2/test.ksv', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( recursive=True, filters=[Filter.exclude('b/2/test.[!ck]sv')], ) ] self.assertEqual(expected, actual) def test_filters_matching_exact_filename(self): data = b'hello world' self.bucket.upload_bytes(data, 'b/a.txt') self.bucket.upload_bytes(data, 'b/b.txt') expected = [ ('b/a.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( recursive=True, filters=[Filter.include('b/a.txt')], ) ] self.assertEqual(expected, actual) expected = [ ('b/b.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( recursive=True, filters=[Filter.exclude('b/a.txt')], ) ] self.assertEqual(expected, actual) def test_filters_mixed_with_wildcards(self): data = b'hello world' self.bucket.upload_bytes(data, 'a.csv') self.bucket.upload_bytes(data, 'a.txt') self.bucket.upload_bytes(data, 'b/a-1.csv') self.bucket.upload_bytes(data, 'b/a-1.txt') self.bucket.upload_bytes(data, 'b/a-2.csv') self.bucket.upload_bytes(data, 'b/a-2.txt') self.bucket.upload_bytes(data, 'b/a-a.csv') self.bucket.upload_bytes(data, 'b/a-a.txt') self.bucket.upload_bytes(data, 'b/a.csv') self.bucket.upload_bytes(data, 'b/a.txt') expected = [ ('a.txt', len(data), 'upload', None), ('b/a-1.txt', len(data), 'upload', None), ('b/a-a.txt', len(data), 'upload', None), ('b/a.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( '*.txt', recursive=True, with_wildcard=True, filters=[Filter.exclude('*-2.txt')], ) ] self.assertEqual(expected, actual) expected = [ ('b/a-1.csv', len(data), 'upload', None), ('b/a-1.txt', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( 'b/?-[1234567890].*', recursive=True, with_wildcard=True, filters=[Filter.exclude('*-2.*')], ) ] self.assertEqual(expected, actual) def test_filters_combination(self): data = b'hello world' self.bucket.upload_bytes(data, 'a.txt') self.bucket.upload_bytes(data, 'b/a-1.csv') self.bucket.upload_bytes(data, 'b/a-1.txt') expected = [ ('a.txt', len(data), 'upload', None), ('b/a-1.csv', len(data), 'upload', None), ] actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket_ls( recursive=True, filters=[Filter.include('b/*'), Filter.exclude('*.txt'), Filter.include('a.txt')], ) ] self.assertEqual(expected, actual) class TestGetFreshState(TestCaseWithBucket): def test_ok(self): same_but_different = self.api.get_bucket_by_id(self.bucket.id_) same_but_different = same_but_different.get_fresh_state() assert isinstance(same_but_different, Bucket) assert id(same_but_different) != id(self.bucket) assert same_but_different.as_dict() == self.bucket.as_dict() same_but_different = same_but_different.update(bucket_info={'completely': 'new info'}) if apiver_deps.V <= 1: same_but_different = BucketFactory.from_api_bucket_dict(self.api, same_but_different) assert same_but_different.as_dict() != self.bucket.as_dict() refreshed_bucket = self.bucket.get_fresh_state() assert same_but_different.as_dict() == refreshed_bucket.as_dict() def test_fail(self): self.api.delete_bucket(self.bucket) with pytest.raises(BucketIdNotFound): self.bucket.get_fresh_state() class TestListVersions(TestCaseWithBucket): def test_single_version(self): data = b'hello world' a_id = self.bucket.upload_bytes(data, 'a').id_ b_id = self.bucket.upload_bytes(data, 'b').id_ c_id = self.bucket.upload_bytes(data, 'c').id_ expected = [(a_id, 'a', 11, 'upload')] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('a') ] self.assertEqual(expected, actual) expected = [(b_id, 'b', 11, 'upload')] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('b') ] self.assertEqual(expected, actual) expected = [(c_id, 'c', 11, 'upload')] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('c') ] self.assertEqual(expected, actual) def test_multiple_version(self): a_id1 = self.bucket.upload_bytes(b'first version', 'a').id_ a_id2 = self.bucket.upload_bytes(b'second version', 'a').id_ a_id3 = self.bucket.upload_bytes(b'last version', 'a').id_ expected = [ (a_id3, 'a', 12, 'upload'), (a_id2, 'a', 14, 'upload'), (a_id1, 'a', 13, 'upload'), ] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('a') ] self.assertEqual(expected, actual) def test_ignores_subdirectory(self): data = b'hello world' file_id = self.bucket.upload_bytes(data, 'a/b').id_ self.bucket.upload_bytes(data, 'a/b/c') expected = [(file_id, 'a/b', 11, 'upload')] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('a/b') ] self.assertEqual(expected, actual) def test_all_versions_in_response(self): data = b'hello world' file_id = self.bucket.upload_bytes(data, 'a/b').id_ self.bucket.upload_bytes(data, 'a/b/c') expected = [(file_id, 'a/b', 11, 'upload')] actual = [ (info.id_, info.file_name, info.size, info.action) for info in self.bucket.list_file_versions('a/b', fetch_count=1) ] self.assertEqual(expected, actual) def test_bad_fetch_count(self): try: # Convert to a list to cause the generator to execute. list(self.bucket.list_file_versions('a', fetch_count=0)) self.fail('should have raised ValueError') except ValueError as e: self.assertEqual('unsupported fetch_count value', str(e)) def test_encryption(self): data = b'hello world' a = self.bucket.upload_bytes(data, 'a') a_id = a.id_ self.assertEqual(a.server_side_encryption, SSE_NONE) b = self.bucket.upload_bytes(data, 'b', encryption=SSE_B2_AES) self.assertEqual(b.server_side_encryption, SSE_B2_AES) b_id = b.id_ # c_id = self.bucket.upload_bytes(data, 'c', encryption=SSE_NONE).id_ # TODO self.bucket.copy(a_id, 'd', destination_encryption=SSE_B2_AES) self.bucket.copy( b_id, 'e', destination_encryption=SSE_C_AES, file_info={}, content_type='text/plain' ) actual = [info.server_side_encryption for info in self.bucket.list_file_versions('a')][0] self.assertEqual(SSE_NONE, actual) # bucket default actual = self.bucket.get_file_info_by_name('a').server_side_encryption self.assertEqual(SSE_NONE, actual) # bucket default actual = [info.server_side_encryption for info in self.bucket.list_file_versions('b')][0] self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 actual = self.bucket.get_file_info_by_name('b').server_side_encryption self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 # actual = [info.server_side_encryption for info in self.bucket.list_file_versions('c')][0] # self.assertEqual(SSE_NONE, actual) # explicitly requested none actual = [info.server_side_encryption for info in self.bucket.list_file_versions('d')][0] self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 actual = self.bucket.get_file_info_by_name('d').server_side_encryption self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 actual = [info.server_side_encryption for info in self.bucket.list_file_versions('e')][0] self.assertEqual(SSE_C_AES_NO_SECRET, actual) # explicitly requested sse-c actual = self.bucket.get_file_info_by_name('e').server_side_encryption self.assertEqual(SSE_C_AES_NO_SECRET, actual) # explicitly requested sse-c class TestCopyFile(TestCaseWithBucket): @classmethod def _copy_function(cls, bucket): if apiver_deps.V <= 1: return bucket.copy_file else: return bucket.copy @pytest.mark.apiver(from_ver=2) def test_copy_big(self): data = b'HelloWorld' * 100 for i in range(10): data += bytes(':#' + str(i) + '$' + 'abcdefghijklmnopqrstuvwx' * 4, 'ascii') file_info = self.bucket.upload_bytes(data, 'file1') self.bucket.copy( file_info.id_, 'file2', min_part_size=200, max_part_size=400, ) self._check_file_contents('file2', data) def test_copy_without_optional_params(self): file_id = self._make_file() if apiver_deps.V <= 1: f = self.bucket.copy_file(file_id, 'hello_new.txt') assert f['action'] == 'copy' else: f = self.bucket.copy(file_id, 'hello_new.txt') assert f.action == 'copy' expected = [('hello.txt', 11, 'upload', None), ('hello_new.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_range(self): file_id = self._make_file() # data = b'hello world' # 3456789 if apiver_deps.V <= 1: self.bucket.copy_file( file_id, 'hello_new.txt', bytes_range=(3, 9), ) # inclusive, confusingly else: self.bucket.copy(file_id, 'hello_new.txt', offset=3, length=7) self._check_file_contents('hello_new.txt', b'lo worl') self._check_large_file_sha1('hello_new.txt', None) expected = [('hello.txt', 11, 'upload', None), ('hello_new.txt', 7, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) @pytest.mark.apiver(to_ver=1) def test_copy_with_invalid_metadata(self): file_id = self._make_file() try: self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.COPY, content_type='application/octet-stream', ) self.fail('should have raised InvalidMetadataDirective') except InvalidMetadataDirective as e: self.assertEqual( 'content_type and file_info should be None when metadata_directive is COPY', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) @pytest.mark.apiver(to_ver=1) def test_copy_with_invalid_metadata_replace(self): file_id = self._make_file() try: self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.REPLACE, ) self.fail('should have raised InvalidMetadataDirective') except InvalidMetadataDirective as e: self.assertEqual( 'content_type cannot be None when metadata_directive is REPLACE', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) @pytest.mark.apiver(to_ver=1) def test_copy_with_replace_metadata(self): file_id = self._make_file() self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.REPLACE, content_type='text/plain', ) expected = [ ('hello.txt', 11, 'upload', 'b2/x-auto', None), ('hello_new.txt', 11, 'upload', 'text/plain', None), ] actual = [ (info.file_name, info.size, info.action, info.content_type, folder) for (info, folder) in self.bucket_ls(show_versions=True) ] self.assertEqual(expected, actual) def test_copy_with_unsatisfied_range(self): file_id = self._make_file() try: if apiver_deps.V <= 1: self.bucket.copy_file( file_id, 'hello_new.txt', bytes_range=(12, 15), ) else: self.bucket.copy( file_id, 'hello_new.txt', offset=12, length=3, ) self.fail('should have raised UnsatisfiableRange') except UnsatisfiableRange as e: self.assertEqual( 'The range in the request is outside the size of the file', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_different_bucket(self): source_bucket = self.api.create_bucket('source-bucket', 'allPublic') file_id = self._make_file(source_bucket) self._copy_function(self.bucket)(file_id, 'hello_new.txt') def ls(bucket): return [ (info.file_name, info.size, info.action, folder) for (info, folder) in bucket_ls(bucket, show_versions=True) ] expected = [('hello.txt', 11, 'upload', None)] self.assertEqual(expected, ls(source_bucket)) expected = [('hello_new.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_retention(self): for data in [self._make_data(self.simulator.MIN_PART_SIZE * 3), b'hello']: for length in [None, len(data)]: with self.subTest(real_length=len(data), length=length): file_id = self.bucket.upload_bytes(data, 'original_file').id_ resulting_file_version = self.bucket.copy( file_id, 'copied_file', file_retention=FileRetentionSetting(RetentionMode.COMPLIANCE, 100), legal_hold=LegalHold.ON, max_part_size=400, ) self.assertEqual( FileRetentionSetting(RetentionMode.COMPLIANCE, 100), resulting_file_version.file_retention, ) self.assertEqual(LegalHold.ON, resulting_file_version.legal_hold) def test_copy_encryption(self): data = b'hello_world' a = self.bucket.upload_bytes(data, 'a') a_id = a.id_ self.assertEqual(a.server_side_encryption, SSE_NONE) b = self.bucket.upload_bytes(data, 'b', encryption=SSE_B2_AES) self.assertEqual(b.server_side_encryption, SSE_B2_AES) b_id = b.id_ c = self.bucket.upload_bytes(data, 'c', encryption=SSE_C_AES) self.assertEqual(c.server_side_encryption, SSE_C_AES_NO_SECRET) c_id = c.id_ for length in [None, len(data)]: for kwargs, expected_encryption in [ (dict(file_id=a_id, destination_encryption=SSE_B2_AES), SSE_B2_AES), ( dict( file_id=a_id, destination_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain', ), SSE_C_AES_NO_SECRET, ), ( dict( file_id=a_id, destination_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain', ), SSE_C_AES_NO_SECRET, ), (dict(file_id=b_id), SSE_NONE), (dict(file_id=b_id, source_encryption=SSE_B2_AES), SSE_NONE), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_B2_AES, ), SSE_B2_AES, ), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain', ), SSE_C_AES_NO_SECRET, ), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain', ), SSE_C_AES_NO_SECRET, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain', ), SSE_NONE, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain', ), SSE_NONE, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES ), SSE_C_AES_NO_SECRET, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_B2_AES, source_file_info={'old': 'value'}, source_content_type='text/plain', ), SSE_B2_AES, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_B2_AES, file_info={'new': 'value'}, content_type='text/plain', ), SSE_B2_AES, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES_2, source_file_info={'old': 'value'}, source_content_type='text/plain', ), SSE_C_AES_2_NO_SECRET, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES_2, file_info={'new': 'value'}, content_type='text/plain', ), SSE_C_AES_2_NO_SECRET, ), ]: with self.subTest(kwargs=kwargs, length=length, data=data): file_info = self.bucket.copy(**kwargs, new_file_name='new_file', length=length) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(file_info.server_side_encryption, expected_encryption) def _make_file(self, bucket=None): data = b'hello world' actual_bucket = bucket or self.bucket return actual_bucket.upload_bytes(data, 'hello.txt').id_ class TestUpdate(TestCaseWithBucket): def test_update(self): result = self.bucket.update( bucket_type='allPrivate', bucket_info={'info': 'o'}, cors_rules={'andrea': 'corr'}, lifecycle_rules=[{'fileNamePrefix': 'is_life'}], default_server_side_encryption=SSE_B2_AES, default_retention=BucketRetentionSetting( RetentionMode.COMPLIANCE, RetentionPeriod(years=7) ), replication=REPLICATION, ) if apiver_deps.V <= 1: self.maxDiff = None with suppress(KeyError): del result['replicationConfiguration'] self.assertEqual( { 'accountId': 'account-0', 'bucketId': 'bucket_0', 'bucketInfo': {'info': 'o'}, 'bucketName': 'my-bucket', 'bucketType': 'allPrivate', 'corsRules': {'andrea': 'corr'}, 'defaultServerSideEncryption': { 'isClientAuthorizedToRead': True, 'value': {'algorithm': 'AES256', 'mode': 'SSE-B2'}, }, 'fileLockConfiguration': { 'isClientAuthorizedToRead': True, 'value': { 'defaultRetention': { 'mode': 'compliance', 'period': {'unit': 'years', 'duration': 7}, }, 'isFileLockEnabled': None, }, }, 'lifecycleRules': [{'fileNamePrefix': 'is_life'}], 'options': set(), 'revision': 2, }, result, ) else: self.assertIsInstance(result, Bucket) assertions_mapping = { 'id_': self.bucket.id_, 'name': self.bucket.name, 'type_': 'allPrivate', 'bucket_info': {'info': 'o'}, 'cors_rules': {'andrea': 'corr'}, 'lifecycle_rules': [{'fileNamePrefix': 'is_life'}], 'options_set': set(), 'default_server_side_encryption': SSE_B2_AES, 'default_retention': BucketRetentionSetting( RetentionMode.COMPLIANCE, RetentionPeriod(years=7) ), 'replication': REPLICATION, } for attr_name, attr_value in assertions_mapping.items(): self.maxDiff = None print('---', attr_name, '---') print(attr_value) print('?=?') print(getattr(result, attr_name)) self.assertEqual(attr_value, getattr(result, attr_name), attr_name) @pytest.mark.apiver(from_ver=2) def test_empty_replication(self): self.bucket.update( replication=ReplicationConfiguration( rules=[], source_to_destination_key_mapping={}, ), ) def test_update_if_revision_is(self): current_revision = self.bucket.revision self.bucket.update( lifecycle_rules=[{'fileNamePrefix': 'is_life'}], if_revision_is=current_revision, ) updated_bucket = self.api.get_bucket_by_name(self.bucket.name) self.assertEqual([{'fileNamePrefix': 'is_life'}], updated_bucket.lifecycle_rules) try: self.bucket.update( lifecycle_rules=[{'fileNamePrefix': 'is_life'}], if_revision_is=current_revision, # this is now the old revision ) except Exception: pass not_updated_bucket = self.api.get_bucket_by_name(self.bucket.name) self.assertEqual([{'fileNamePrefix': 'is_life'}], not_updated_bucket.lifecycle_rules) def test_is_file_lock_enabled(self): assert not self.bucket.is_file_lock_enabled # set is_file_lock_enabled to False when it's already false self.bucket.update(is_file_lock_enabled=False) updated_bucket = self.api.get_bucket_by_name(self.bucket.name) assert not updated_bucket.is_file_lock_enabled # sunny day scenario self.bucket.update(is_file_lock_enabled=True) updated_bucket = self.api.get_bucket_by_name(self.bucket.name) assert updated_bucket.is_file_lock_enabled assert self.simulator.bucket_name_to_bucket[self.bucket.name].is_file_lock_enabled # attempt to clear is_file_lock_enabled with pytest.raises(DisablingFileLockNotSupported): self.bucket.update(is_file_lock_enabled=False) updated_bucket = self.api.get_bucket_by_name(self.bucket.name) assert updated_bucket.is_file_lock_enabled # attempt to set is_file_lock_enabled when it's already set self.bucket.update(is_file_lock_enabled=True) updated_bucket = self.api.get_bucket_by_name(self.bucket.name) assert updated_bucket.is_file_lock_enabled @pytest.mark.apiver(from_ver=2) def test_is_file_lock_enabled_source_replication(self): assert not self.bucket.is_file_lock_enabled # attempt to set is_file_lock_enabled with source replication enabled self.bucket.update(replication=REPLICATION) with pytest.raises(SourceReplicationConflict): self.bucket.update(is_file_lock_enabled=True) updated_bucket = self.bucket.update(replication=REPLICATION) assert not updated_bucket.is_file_lock_enabled # sunny day scenario self.bucket.update( replication=ReplicationConfiguration( rules=[], source_to_destination_key_mapping={}, ) ) self.bucket.update(is_file_lock_enabled=True) updated_bucket = self.api.get_bucket_by_name(self.bucket.name) assert updated_bucket.is_file_lock_enabled assert self.simulator.bucket_name_to_bucket[self.bucket.name].is_file_lock_enabled class TestUpload(TestCaseWithBucket): def test_upload_bytes(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1') self.assertTrue(isinstance(file_info, VFileVersionInfo)) self._check_file_contents('file1', data) self._check_large_file_sha1('file1', None) self.assertEqual(file_info.server_side_encryption, SSE_NONE) def test_upload_bytes_file_retention(self): data = b'hello world' retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 150) file_info = self.bucket.upload_bytes( data, 'file1', file_retention=retention, legal_hold=LegalHold.ON ) self._check_file_contents('file1', data) self._check_large_file_sha1('file1', None) self.assertEqual(retention, file_info.file_retention) self.assertEqual(LegalHold.ON, file_info.legal_hold) def test_upload_bytes_sse_b2(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1', encryption=SSE_B2_AES) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(file_info.server_side_encryption, SSE_B2_AES) def test_upload_bytes_sse_c(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1', encryption=SSE_C_AES) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(SSE_C_AES_NO_SECRET, file_info.server_side_encryption) def test_upload_local_file_sse_b2(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1', encryption=SSE_B2_AES) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(file_info.server_side_encryption, SSE_B2_AES) self._check_file_contents('file1', data) def test_upload_local_file_sse_c(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1', encryption=SSE_C_AES) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(SSE_C_AES_NO_SECRET, file_info.server_side_encryption) self._check_file_contents('file1', data) def test_upload_local_file_retention(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 150) file_info = self.bucket.upload_local_file( path, 'file1', encryption=SSE_C_AES, file_retention=retention, legal_hold=LegalHold.ON, ) self._check_file_contents('file1', data) self.assertEqual(retention, file_info.file_retention) self.assertEqual(LegalHold.ON, file_info.legal_hold) def test_upload_local_file_cache_control(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file( path, 'file1', encryption=SSE_C_AES, cache_control='max-age=3600' ) self._check_file_contents('file1', data) self.assertEqual(file_info.cache_control, 'max-age=3600') def test_upload_bytes_cache_control(self): data = b'hello world' file_info = self.bucket.upload_bytes( data, 'file1', encryption=SSE_C_AES, cache_control='max-age=3600' ) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(file_info.cache_control, 'max-age=3600') def test_upload_bytes_progress(self): data = b'hello world' progress_listener = StubProgressListener() self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertTrue(progress_listener.is_valid()) def test_upload_local_file(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1') self._check_file_contents('file1', data) self._check_large_file_sha1('file1', None) self.assertTrue(isinstance(file_info, VFileVersionInfo)) self.assertEqual(file_info.server_side_encryption, SSE_NONE) print(file_info.as_dict()) self.assertEqual(file_info.as_dict()['serverSideEncryption'], {'mode': 'none'}) @pytest.mark.apiver(from_ver=2) def test_upload_local_file_incremental(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') small_data = b'Hello world!' big_data = self._make_data(self.simulator.MIN_PART_SIZE * 3) DATA = [ big_data, big_data + small_data, big_data + small_data + big_data, small_data, small_data + small_data, small_data.upper() + small_data, ] last_data = None for data in DATA: # figure out if this particular upload should be incremental should_be_incremental = ( last_data and data.startswith(last_data) and len(last_data) >= self.simulator.MIN_PART_SIZE ) # if it's incremental, then there should be two sources concatenated, otherwise one expected_source_count = 2 if should_be_incremental else 1 # is the result file expected to be a large file expected_large_file = ( should_be_incremental or len(data) > self.simulator.MIN_PART_SIZE ) expected_parts_sizes = ( [len(last_data), len(data) - len(last_data)] if should_be_incremental else [len(data)] ) write_file(path, data) with mock.patch.object( self.bucket, 'concatenate', wraps=self.bucket.concatenate ) as mocked_concatenate: self.bucket.upload_local_file(path, 'file1', upload_mode=UploadMode.INCREMENTAL) mocked_concatenate.assert_called_once() call = mocked_concatenate.mock_calls[0] # TODO: use .args[0] instead of [1][0] when we drop Python 3.7 assert len(call[1][0]) == expected_source_count # Ensuring that the part sizes make sense. parts_sizes = [entry.get_content_length() for entry in call[1][0]] assert parts_sizes == expected_parts_sizes if should_be_incremental: # Ensuring that the first part is a copy. # Order of indices: pick arguments, pick first argument, first element of the first argument. self.assertIsInstance(call[1][0][0], CopySource) self._check_file_contents('file1', data) if expected_large_file: self._check_large_file_sha1('file1', hex_sha1_of_bytes(data)) last_data = data @pytest.mark.skipif(platform.system() == 'Windows', reason='no os.mkfifo() on Windows') def test_upload_fifo(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') os.mkfifo(path) with self.assertRaises(InvalidUploadSource): self.bucket.upload_local_file(path, 'file1') @pytest.mark.skipif(platform.system() == 'Windows', reason='no os.symlink() on Windows') def test_upload_dead_symlink(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') os.symlink('non-existing', path) with self.assertRaises(InvalidUploadSource): self.bucket.upload_local_file(path, 'file1') def test_upload_local_wrong_sha(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file123') data = b'hello world' write_file(path, data) with self.assertRaises(FileSha1Mismatch): self.bucket.upload_local_file( path, 'file123', sha1_sum='abcd' * 10, ) def test_upload_one_retryable_error(self): self.simulator.set_upload_errors([CanRetry(True)]) data = b'hello world' self.bucket.upload_bytes(data, 'file1') def test_upload_timeout(self): self.simulator.set_upload_errors([B2RequestTimeoutDuringUpload()]) data = b'hello world' self.bucket.upload_bytes(data, 'file1') def test_upload_file_one_fatal_error(self): self.simulator.set_upload_errors([CanRetry(False)]) data = b'hello world' with self.assertRaises(CanRetry): self.bucket.upload_bytes(data, 'file1') def test_upload_file_too_many_retryable_errors(self): self.simulator.set_upload_errors([CanRetry(True)] * 6) data = b'hello world' with self.assertRaises(MaxRetriesExceeded): self.bucket.upload_bytes(data, 'file1') def test_upload_large(self): data = self._make_data(self.simulator.MIN_PART_SIZE * 3) progress_listener = StubProgressListener() self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self._check_file_contents('file1', data) self._check_large_file_sha1('file1', hex_sha1_of_bytes(data)) self.assertTrue(progress_listener.is_valid()) def test_upload_local_large_file(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = self._make_data(self.simulator.MIN_PART_SIZE * 3) write_file(path, data) self.bucket.upload_local_file(path, 'file1') self._check_file_contents('file1', data) self._check_large_file_sha1('file1', hex_sha1_of_bytes(data)) def test_upload_local_large_file_over_10k_parts(self): pytest.skip('this test is really slow and impedes development') # TODO: fix it with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = self._make_data(self.simulator.MIN_PART_SIZE * 10001) # 2MB on the simulator write_file(path, data) self.bucket.upload_local_file(path, 'file1') self._check_file_contents('file1', data) self._check_large_file_sha1('file1', hex_sha1_of_bytes(data)) def test_create_file_over_10k_parts(self): data = b'hello world' * 20000 f1_id = self.bucket.upload_bytes(data, 'f1').id_ with tempfile.TemporaryDirectory(): write_intents = [ WriteIntent( CopySource(f1_id, length=len(data), offset=0), destination_offset=0, ) ] * 10 created_file = self.bucket.create_file( write_intents, file_name='created_file', min_part_size=10, max_part_size=200, ) self.assertIsInstance(created_file, VFileVersionInfo) actual = ( created_file.id_, created_file.file_name, created_file.size, created_file.server_side_encryption, ) expected = ('9998', 'created_file', len(data), SSE_NONE) self.assertEqual(expected, actual) def test_upload_large_resume(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_no_parts(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_all_parts_there(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[:part_size]) self._upload_part(large_file_id, 2, data[part_size : 2 * part_size]) self._upload_part(large_file_id, 3, data[2 * part_size :]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_part_does_not_match(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 3, data[:part_size]) # wrong part number for this data progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_wrong_part_size(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[: part_size + 1]) # one byte to much progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_file_info(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1', {'property': 'value1'}) self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes( data, 'file1', progress_listener=progress_listener, file_info={'property': 'value1'} ) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_file_info_does_not_match(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1', {'property': 'value1'}) self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes( data, 'file1', progress_listener=progress_listener, file_info={'property': 'value2'} ) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_file_with_restricted_api_key(self): self.simulator.key_id_to_key[self.account_id].name_prefix_or_none = 'path/to' part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes( data, 'path/to/file1', progress_listener=progress_listener ) self.assertEqual(len(data), file_info.size) self._check_file_contents('path/to/file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_stream(self): data = self._make_data(self.simulator.MIN_PART_SIZE * 3) self.bucket.upload_unbound_stream(io.BytesIO(data), 'file1') self._check_file_contents('file1', data) def test_upload_stream_from_file(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = self._make_data(self.simulator.MIN_PART_SIZE * 3) write_file(path, data) with open(path, 'rb') as f: self.bucket.upload_unbound_stream(f, 'file1') self._check_file_contents('file1', data) def _start_large_file(self, file_name, file_info=None): if file_info is None: file_info = {} large_file_info = self.simulator.start_large_file( self.api_url, self.account_auth_token, self.bucket_id, file_name, None, file_info ) return large_file_info['fileId'] def _upload_part(self, large_file_id, part_number, part_data): part_stream = BytesIO(part_data) upload_info = self.simulator.get_upload_part_url( self.api_url, self.account_auth_token, large_file_id ) self.simulator.upload_part( upload_info['uploadUrl'], upload_info['authorizationToken'], part_number, len(part_data), hex_sha1_of_bytes(part_data), part_stream, ) class TestBucketRaisingSession(TestUpload): def get_api(self): class B2SessionRaising(B2Session): def __init__(self, *args, **kwargs): self._raise_count = 0 self._raise_until = 1 super().__init__(*args, **kwargs) def upload_part( self, file_id, part_number, content_length, sha1_sum, input_stream, server_side_encryption=None, ): if self._raise_count < self._raise_until: self._raise_count += 1 raise B2ConnectionError() return super().upload_part( file_id, part_number, content_length, sha1_sum, input_stream, server_side_encryption, ) class B2ApiPatched(B2Api): SESSION_CLASS = staticmethod(B2SessionRaising) self.api = B2ApiPatched( self.account_info, cache=self.CACHE_CLASS(), api_config=B2HttpApiConfig(_raw_api_class=self.RAW_SIMULATOR_CLASS), ) return self.api def test_upload_chunk_retry_stream_open(self): assert self.api.session._raise_count == 0 data = self._make_data(self.simulator.MIN_PART_SIZE * 3) self.bucket.upload_unbound_stream(io.BytesIO(data), 'file1') self._check_file_contents('file1', data) assert self.api.session._raise_count == 1 def test_upload_chunk_stream_guard_closes(self): data = self._make_data(self.simulator.MIN_PART_SIZE * 3) large_file_upload_state = mock.MagicMock() large_file_upload_state.has_error.return_value = False class TrackedUploadSourceBytes(UploadSourceBytes): def __init__(self, *args, **kwargs): self._close_called = 0 super().__init__(*args, **kwargs) def open(self): class TrackedBytesIO(io.BytesIO): def __init__(self, parent, *args, **kwargs): self._parent = parent super().__init__(*args, **kwargs) def close(self): self._parent._close_called += 1 return super().close() return TrackedBytesIO(self, self.data_bytes) data_source = TrackedUploadSourceBytes(data) assert data_source._close_called == 0 file_id = self._start_large_file('file1') self.api.services.upload_manager.upload_part( self.bucket_id, file_id, data_source, 1, large_file_upload_state ).result() # one retry means two potential callback calls, but we want one only assert data_source._close_called == 1 class TestConcatenate(TestCaseWithBucket): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.concatenate(sources, file_name=file_name, encryption=encryption) def test_create_remote(self): data = b'hello world' f1_id = self.bucket.upload_bytes(data, 'f1').id_ f2_id = self.bucket.upload_bytes(data, 'f1').id_ with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file') write_file(path, data) created_file = self._create_remote( [ CopySource(f1_id, length=len(data), offset=0), UploadSourceLocalFile(path), CopySource(f2_id, length=len(data), offset=0), ], file_name='created_file', ) self.assertIsInstance(created_file, VFileVersionInfo) actual = ( created_file.id_, created_file.file_name, created_file.size, created_file.server_side_encryption, ) expected = ('9997', 'created_file', 33, SSE_NONE) self.assertEqual(expected, actual) def test_create_remote_encryption(self): for data in [b'hello_world', self._make_data(self.simulator.MIN_PART_SIZE * 3)]: f1_id = self.bucket.upload_bytes(data, 'f1', encryption=SSE_C_AES).id_ f2_id = self.bucket.upload_bytes(data, 'f1', encryption=SSE_C_AES_2).id_ with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file') write_file(path, data) created_file = self._create_remote( [ CopySource(f1_id, length=len(data), offset=0, encryption=SSE_C_AES), UploadSourceLocalFile(path), CopySource(f2_id, length=len(data), offset=0, encryption=SSE_C_AES_2), ], file_name=f'created_file_{len(data)}', encryption=SSE_C_AES, ) self.assertIsInstance(created_file, VFileVersionInfo) actual = ( created_file.id_, created_file.file_name, created_file.size, created_file.server_side_encryption, ) expected = ( mock.ANY, f'created_file_{len(data)}', mock.ANY, # FIXME: this should be equal to len(data) * 3, # but there is a problem in the simulator/test code somewhere SSE_C_AES_NO_SECRET, ) self.assertEqual(expected, actual) class TestCreateFile(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.create_file( [wi for wi in WriteIntent.wrap_sources_iterator(sources)], file_name=file_name, encryption=encryption, ) class TestConcatenateStream(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.concatenate_stream(sources, file_name=file_name, encryption=encryption) class TestCreateFileStream(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.create_file_stream( [wi for wi in WriteIntent.wrap_sources_iterator(sources)], file_name=file_name, encryption=encryption, ) class TestCustomTimestamp(TestCaseWithBucket): def test_custom_timestamp(self): data = b'hello world' # upload self.bucket.upload_bytes(data, 'file0', custom_upload_timestamp=0) with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') write_file(path, data) self.bucket.upload_local_file(path, 'file1', custom_upload_timestamp=1) upload_source = UploadSourceBytes(data) self.bucket.upload(upload_source, 'file2', custom_upload_timestamp=2) self.bucket.upload_unbound_stream(io.BytesIO(data), 'file3', custom_upload_timestamp=3) # concatenate self.bucket.concatenate([upload_source], 'file4', custom_upload_timestamp=4) self.bucket.concatenate_stream([upload_source], 'file5', custom_upload_timestamp=5) # create_file self.bucket.create_file( [WriteIntent(upload_source, destination_offset=0)], 'file6', custom_upload_timestamp=6 ) self.bucket.create_file_stream( [WriteIntent(upload_source, destination_offset=0)], 'file7', custom_upload_timestamp=7 ) def ls(bucket): return [(info.file_name, info.upload_timestamp) for (info, folder) in bucket_ls(bucket)] expected = [ ('file0', 0), ('file1', 1), ('file2', 2), ('file3', 3), ('file4', 4), ('file5', 5), ('file6', 6), ('file7', 7), ] self.assertEqual(ls(self.bucket), expected) class DownloadTestsBase: DATA = NotImplemented def setUp(self): super().setUp() self.file_version = self.bucket.upload_bytes(self.DATA.encode(), 'file1') self.encrypted_file_version = self.bucket.upload_bytes( self.DATA.encode(), 'enc_file1', encryption=SSE_C_AES ) self.bytes_io = io.BytesIO() if apiver_deps.V <= 1: self.download_dest = DownloadDestBytes() else: self.download_dest = None self.progress_listener = StubProgressListener() def _verify(self, expected_result, check_progress_listener=True): self._assert_downloaded_data(expected_result) if check_progress_listener: valid, reason = self.progress_listener.is_valid_reason( check_progress=False, check_monotonic_progress=True, ) assert valid, reason def _assert_downloaded_data(self, expected_result): if apiver_deps.V <= 1: assert self.download_dest.get_bytes_written() == expected_result.encode() else: assert self.bytes_io.getvalue() == expected_result.encode() def download_file_by_id(self, file_id, v1_download_dest=None, v2_file=None, **kwargs): if apiver_deps.V <= 1: self.bucket.download_file_by_id( file_id, v1_download_dest or self.download_dest, **kwargs ) else: self.bucket.download_file_by_id(file_id, **kwargs).save(v2_file or self.bytes_io) def download_file_by_name(self, file_name, download_dest=None, **kwargs): if apiver_deps.V <= 1: self.bucket.download_file_by_name( file_name, download_dest or self.download_dest, **kwargs ) else: self.bucket.download_file_by_name(file_name, **kwargs).save(self.bytes_io) class TestDownloadException(DownloadTestsBase, TestCaseWithBucket): DATA = 'some data' def test_download_file_by_name(self): if apiver_deps.V <= 1: exception_class = AssertionError else: exception_class = ValueError with mock.patch.object(self.bucket.api.services.download_manager, 'strategies', new=[]): with pytest.raises(exception_class) as exc_info: self.download_file_by_name(self.file_version.file_name) assert str(exc_info.value) == 'no strategy suitable for download was found!' class DownloadTests(DownloadTestsBase): DATA = 'abcdefghijklmnopqrs' def test_v2_return_types(self): download_kwargs = { 'range_': (7, 18), 'encryption': SSE_C_AES, 'progress_listener': self.progress_listener, } file_version = self.bucket.upload_bytes( self.DATA.encode(), 'enc_file2', encryption=SSE_C_AES ) other_properties = { 'download_version': DownloadVersion( api=self.api, id_=file_version.id_, file_name=file_version.file_name, size=len(self.DATA), content_type=file_version.content_type, content_sha1=file_version.content_sha1, file_info=file_version.file_info, upload_timestamp=file_version.upload_timestamp, server_side_encryption=file_version.server_side_encryption, range_=Range(7, 18), content_disposition=None, content_length=12, content_language=None, expires=None, cache_control=None, content_encoding=None, file_retention=file_version.file_retention, legal_hold=file_version.legal_hold, ), } ret = self.bucket.download_file_by_id(file_version.id_, **download_kwargs) assert isinstance(ret, DownloadedFile), type(ret) for attr_name, expected_value in {**download_kwargs, **other_properties}.items(): assert getattr(ret, attr_name) == expected_value, attr_name if apiver_deps.V >= 2: ret = self.bucket.download_file_by_name(file_version.file_name, **download_kwargs) assert isinstance(ret, DownloadedFile), type(ret) for attr_name, expected_value in {**download_kwargs, **other_properties}.items(): assert getattr(ret, attr_name) == expected_value, attr_name ret = file_version.download(**download_kwargs) assert isinstance(ret, DownloadedFile), type(ret) for attr_name, expected_value in {**download_kwargs, **other_properties}.items(): assert getattr(ret, attr_name) == expected_value, attr_name @pytest.mark.apiver(to_ver=1) def test_v1_return_types(self): expected = { 'contentLength': 19, 'contentSha1': '893e69ff0109f3459c4243013b3de8b12b41a30e', 'contentType': 'b2/x-auto', 'fileId': '9999', 'fileInfo': {}, 'fileName': 'file1', } ret = self.bucket.download_file_by_id(self.file_version.id_, self.download_dest) assert ret == expected ret = self.bucket.download_file_by_name(self.file_version.file_name, self.download_dest) assert ret == expected def test_download_file_version(self): self.file_version.download().save(self.bytes_io) assert self.bytes_io.getvalue() == self.DATA.encode() # self._verify performs different checks based on apiver, # but this is a new feature so it works the same on v2, v1 and v0 def test_download_by_id_no_progress(self): self.download_file_by_id(self.file_version.id_) self._verify(self.DATA, check_progress_listener=False) def test_download_by_name_no_progress(self): self.download_file_by_name('file1') self._verify(self.DATA, check_progress_listener=False) def test_download_by_name_progress(self): self.download_file_by_name('file1', progress_listener=self.progress_listener) self._verify(self.DATA) def test_download_by_id_progress(self): self.download_file_by_id(self.file_version.id_, progress_listener=self.progress_listener) self._verify(self.DATA) def test_download_by_id_progress_partial(self): self.download_file_by_id( self.file_version.id_, progress_listener=self.progress_listener, range_=(3, 9) ) self._verify('defghij') def test_download_by_id_progress_exact_range(self): self.download_file_by_id( self.file_version.id_, progress_listener=self.progress_listener, range_=(0, 18) ) self._verify(self.DATA) def test_download_by_id_progress_range_one_off(self): with self.assertRaises( InvalidRange, msg='A range of 0-19 was requested (size of 20), but cloud could only serve 19 of that', ): self.download_file_by_id( self.file_version.id_, progress_listener=self.progress_listener, range_=(0, 19), ) @pytest.mark.apiver(to_ver=1) def test_download_by_id_progress_partial_inplace_overwrite_v1(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # ||||||| # vvvvvvv # # 123defghij1234567890 with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file2') download_dest = PreSeekedDownloadDest(seek_target=3, local_file_path=path) data = b'12345678901234567890' write_file(path, data) self.download_file_by_id( self.file_version.id_, download_dest, progress_listener=self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'123defghij1234567890') @pytest.mark.apiver(from_ver=2) def test_download_by_id_progress_partial_inplace_overwrite_v2(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # ||||||| # vvvvvvv # # 123defghij1234567890 with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file2') data = b'12345678901234567890' write_file(path, data) with open(path, 'rb+') as file: file.seek(3) self.download_file_by_id( self.file_version.id_, v2_file=file, progress_listener=self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'123defghij1234567890') @pytest.mark.apiver(from_ver=2) def test_download_update_mtime_v2(self): with tempfile.TemporaryDirectory() as d: file_version = self.bucket.upload_bytes( self.DATA.encode(), 'file1', file_info={'src_last_modified_millis': '1000'} ) path = os.path.join(d, 'file2') self.bucket.download_file_by_id(file_version.id_).save_to(path) assert pytest.approx(1, rel=0.001) == os.path.getmtime(path) @pytest.mark.apiver(to_ver=1) def test_download_by_id_progress_partial_shifted_overwrite_v1(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # ||||||| # vvvvvvv # # 1234567defghij567890 with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file2') download_dest = PreSeekedDownloadDest(seek_target=7, local_file_path=path) data = b'12345678901234567890' write_file(path, data) self.download_file_by_id( self.file_version.id_, download_dest, progress_listener=self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'1234567defghij567890') @pytest.mark.apiver(from_ver=2) def test_download_by_id_progress_partial_shifted_overwrite_v2(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # ||||||| # vvvvvvv # # 1234567defghij567890 with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file2') data = b'12345678901234567890' write_file(path, data) with open(path, 'rb+') as file: file.seek(7) self.download_file_by_id( self.file_version.id_, v2_file=file, progress_listener=self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'1234567defghij567890') def test_download_by_id_no_progress_encryption(self): self.download_file_by_id(self.encrypted_file_version.id_, encryption=SSE_C_AES) self._verify(self.DATA, check_progress_listener=False) def test_download_by_id_no_progress_wrong_encryption(self): with self.assertRaises(SSECKeyError): self.download_file_by_id(self.encrypted_file_version.id_, encryption=SSE_C_AES_2) def _check_local_file_contents(self, path, expected_contents): with open(path, 'rb') as f: contents = f.read() self.assertEqual(contents, expected_contents) @pytest.mark.apiver(from_ver=2) def test_download_to_non_seekable_file(self): file_version = self.bucket.upload_bytes(self.DATA.encode(), 'file1') non_seekable_strategies = [ strat for strat in self.bucket.api.services.download_manager.strategies if not isinstance(strat, ParallelDownloader) ] context = ( contextlib.nullcontext() if non_seekable_strategies else pytest.raises( ValueError, match='no strategy suitable for download was found!', ) ) output_file = NonSeekableIO() with context: self.download_file_by_id( file_version.id_, v2_file=output_file, ) assert output_file.getvalue() == self.DATA.encode() @pytest.mark.apiver(from_ver=2) def test_download_to_seekable_but_no_read_file(self): file_version = self.bucket.upload_bytes(self.DATA.encode(), 'file1') non_seekable_strategies = [ strat for strat in self.bucket.api.services.download_manager.strategies if not isinstance(strat, ParallelDownloader) ] context = ( contextlib.nullcontext() if non_seekable_strategies else pytest.raises( ValueError, match='no strategy suitable for download was found!', ) ) output_file = io.BytesIO() seekable_but_not_readable = io.BufferedWriter(output_file) # test sanity check assert seekable_but_not_readable.seekable() with pytest.raises(io.UnsupportedOperation): seekable_but_not_readable.read(0) with context: self.download_file_by_id( file_version.id_, v2_file=seekable_but_not_readable, ) seekable_but_not_readable.flush() assert output_file.getvalue() == self.DATA.encode() # download empty file class EmptyFileDownloadScenarioMixin: """use with DownloadTests, but not for TestDownloadParallel as it does not like empty files""" def test_download_by_name_empty_file(self): self.file_version = self.bucket.upload_bytes(b'', 'empty') self.download_file_by_name('empty', progress_listener=self.progress_listener) self._verify('') class UnverifiedChecksumDownloadScenarioMixin: """use with DownloadTests""" def test_download_by_name_unverified_checksum(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1') simulated_file = list(self.simulator.bucket_name_to_bucket.values())[0].file_id_to_file[ file_info.id_ ] simulated_file.content_sha1 = 'unverified:' + simulated_file.content_sha1 # , sha1_sum='unverified:2aae6c35c94fcfb415dbe95f408b9ce91ee846ed') self.download_file_by_name('file1', progress_listener=self.progress_listener) self._verify('hello world') # actual tests # test choosing strategy @pytest.mark.apiver(from_ver=2) class TestChooseStrategy(TestCaseWithBucket): def test_choose_strategy(self): file_version = self.bucket.upload_bytes(b'hello world' * 8, 'file1') download_manager = self.bucket.api.services.download_manager parallel_downloader = ParallelDownloader( force_chunk_size=1, max_streams=32, min_part_size=16, thread_pool=download_manager._thread_pool, ) simple_downloader = download_manager.strategies[1] download_manager.strategies = [ parallel_downloader, simple_downloader, ] with io.BytesIO() as bytes_io: downloaded_file = self.bucket.download_file_by_id(file_version.id_) downloaded_file.save(bytes_io, allow_seeking=True) assert downloaded_file.download_strategy == parallel_downloader downloaded_file = self.bucket.download_file_by_id(file_version.id_) downloaded_file.save(bytes_io, allow_seeking=False) assert downloaded_file.download_strategy == simple_downloader downloaded_file = self.bucket.download_file_by_name(file_version.file_name) downloaded_file.save(bytes_io, allow_seeking=True) assert downloaded_file.download_strategy == parallel_downloader downloaded_file = self.bucket.download_file_by_name(file_version.file_name) downloaded_file.save(bytes_io, allow_seeking=False) assert downloaded_file.download_strategy == simple_downloader # Default tests class TestDownloadDefault(DownloadTests, EmptyFileDownloadScenarioMixin, TestCaseWithBucket): pass class TestDownloadSimple( DownloadTests, UnverifiedChecksumDownloadScenarioMixin, EmptyFileDownloadScenarioMixin, TestCaseWithBucket, ): def setUp(self): super().setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [SimpleDownloader(force_chunk_size=20)] class TestDownloadParallel( DownloadTests, UnverifiedChecksumDownloadScenarioMixin, TestCaseWithBucket, ): def setUp(self): super().setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [ ParallelDownloader( force_chunk_size=2, max_streams=999, min_part_size=2, ), ] class TestDownloadParallelALotOfStreams(DownloadTestsBase, TestCaseWithBucket): DATA = ''.join(['01234567890abcdef'] * 32) def setUp(self): super().setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [ # this should produce 32 streams with 16 single byte writes # so we increase probability of non-sequential writes as much as possible # with great help from random sleeps in FakeResponse ParallelDownloader( force_chunk_size=1, max_streams=32, min_part_size=16, thread_pool=download_manager._thread_pool, ), ] def test_download_by_id_progress_monotonic(self): self.download_file_by_id(self.file_version.id_, progress_listener=self.progress_listener) self._verify(self.DATA) # Truncated downloads class TruncatedFakeResponse(FakeResponse): """ A special FakeResponse class which returns only the first 4 bytes of data. Use it to test followup retries for truncated download issues. """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.data_bytes = self.data_bytes[:4] class TruncatedDownloadBucketSimulator(BucketSimulator): RESPONSE_CLASS = TruncatedFakeResponse class TruncatedDownloadRawSimulator(RawSimulator): BUCKET_SIMULATOR_CLASS = TruncatedDownloadBucketSimulator class TestCaseWithTruncatedDownloadBucket(TestCaseWithBucket): RAW_SIMULATOR_CLASS = TruncatedDownloadRawSimulator ####### actual tests of truncated downloads class TestTruncatedDownloadSimple(DownloadTests, TestCaseWithTruncatedDownloadBucket): def setUp(self): super().setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [SimpleDownloader(force_chunk_size=20)] class TestTruncatedDownloadParallel(DownloadTests, TestCaseWithTruncatedDownloadBucket): def setUp(self): super().setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [ ParallelDownloader( force_chunk_size=3, max_streams=2, min_part_size=2, ) ] class DummyDownloader(AbstractDownloader): def download(self, *args, **kwargs): pass @pytest.mark.parametrize( 'min_chunk_size,max_chunk_size,content_length,align_factor,expected_chunk_size', [ (10, 100, 1000 * 9, 8, 8), # min_chunk_size aligned (10, 100, 1000 * 17, 8, 16), # content_length // 1000 aligned (10, 100, 1000 * 108, 8, 96), # max_chunk_size // 1000 aligned (10, 100, 1000 * 9, 100, 100), # max_chunk_size/align_factor (10, 100, 1000 * 17, 100, 100), # max_chunk_size/align_factor (10, 100, 1000 * 108, 100, 100), # max_chunk_size/align_factor (10, 100, 1, 100, 100), # max_chunk_size/align_factor ], ) def test_downloader_get_chunk_size( min_chunk_size, max_chunk_size, content_length, align_factor, expected_chunk_size ): downloader = DummyDownloader( min_chunk_size=min_chunk_size, max_chunk_size=max_chunk_size, align_factor=align_factor, ) assert downloader._get_chunk_size(content_length) == expected_chunk_size @pytest.mark.apiver(from_ver=2) class TestDownloadTuneWriteBuffer(DownloadTestsBase, TestCaseWithBucket): ALIGN_FACTOR = 123 DATA = 'abc' * 4096 def get_api(self): return B2Api( self.account_info, api_config=B2HttpApiConfig(_raw_api_class=self.RAW_SIMULATOR_CLASS), save_to_buffer_size=self.ALIGN_FACTOR, ) def test_get_chunk_size_alignment(self): download_manager = self.bucket.api.services.download_manager for downloader in download_manager.strategies: assert downloader._get_chunk_size(len(self.DATA)) % self.ALIGN_FACTOR == 0 def test_buffering_in_save_to(self): with tempfile.TemporaryDirectory() as d: path = pathlib.Path(d) / 'file2' with mock.patch('b2sdk._internal.transfer.inbound.downloaded_file.open') as mock_open: mock_open.side_effect = open self.bucket.download_file_by_id(self.file_version.id_).save_to(path) mock_open.assert_called_once_with(path, mock.ANY, buffering=self.ALIGN_FACTOR) assert path.read_text() == self.DATA def test_set_write_buffer_parallel_called_get_chunk_size(self): self._check_called_on_downloader( ParallelDownloader( force_chunk_size=len(self.DATA) // 3, max_streams=999, min_part_size=len(self.DATA) // 3, ) ) def test_set_write_buffer_simple_called_get_chunk_size(self): self._check_called_on_downloader(SimpleDownloader(force_chunk_size=len(self.DATA) // 3)) def _check_called_on_downloader(self, downloader): download_manager = self.bucket.api.services.download_manager download_manager.strategies = [downloader] orig_get_chunk_size = downloader._get_chunk_size with mock.patch.object(downloader, '_get_chunk_size') as mock_get_chunk_size: mock_get_chunk_size.side_effect = orig_get_chunk_size self.download_file_by_id(self.file_version.id_) self._verify(self.DATA, check_progress_listener=False) assert mock_get_chunk_size.called @pytest.mark.apiver(from_ver=2) class TestDownloadNoHashChecking(DownloadTestsBase, TestCaseWithBucket): DATA = 'abcdefghijklmnopqrs' def get_api(self): return B2Api( self.account_info, api_config=B2HttpApiConfig(_raw_api_class=self.RAW_SIMULATOR_CLASS), check_download_hash=False, ) def test_download_by_id_no_hash_checking(self): downloaded_file = self.bucket.download_file_by_id(self.file_version.id_) orig_validate_download = downloaded_file._validate_download with mock.patch.object(downloaded_file, '_validate_download') as mocked_validate_download: mocked_validate_download.side_effect = orig_validate_download downloaded_file.save(self.bytes_io) self._verify(self.DATA, check_progress_listener=False) mocked_validate_download.assert_called_once_with(mock.ANY, '') assert downloaded_file.download_version.content_sha1 != 'none' assert downloaded_file.download_version.content_sha1 != '' class DecodeTestsBase: def setUp(self): super().setUp() self.bucket.upload_bytes( b'Test File 1', 'test.txt?foo=bar', file_info={'custom_info': 'aaa?bbb'} ) self.bucket.upload_bytes( b'Test File 2', 'test.txt%3Ffoo=bar', file_info={'custom_info': 'aaa%3Fbbb'} ) self.bucket.upload_bytes(b'Test File 3', 'test.txt%3Ffoo%3Dbar') self.bucket.upload_bytes(b'Test File 4', 'test.txt%253Ffoo%253Dbar') self.bytes_io = io.BytesIO() if apiver_deps.V <= 1: self.download_dest = DownloadDestBytes() else: self.download_dest = None self.progress_listener = StubProgressListener() def _verify(self, expected_result, check_progress_listener=True): self._assert_downloaded_data(expected_result) if check_progress_listener: valid, reason = self.progress_listener.is_valid_reason( check_progress=False, check_monotonic_progress=True, ) assert valid, reason def _assert_downloaded_data(self, expected_result): if apiver_deps.V <= 1: assert self.download_dest.get_bytes_written() == expected_result.encode() else: assert self.bytes_io.getvalue() == expected_result.encode() def download_file_by_name(self, file_name, download_dest=None, **kwargs): if apiver_deps.V <= 1: self.bucket.download_file_by_name( file_name, download_dest or self.download_dest, **kwargs ) else: self.bucket.download_file_by_name(file_name, **kwargs).save(self.bytes_io) class DecodeTests(DecodeTestsBase, TestCaseWithBucket): def test_file_content_1(self): self.download_file_by_name('test.txt?foo=bar', progress_listener=self.progress_listener) self._verify('Test File 1') def test_file_content_2(self): self.download_file_by_name('test.txt%3Ffoo=bar', progress_listener=self.progress_listener) self._verify('Test File 2') def test_file_content_3(self): self.download_file_by_name('test.txt%3Ffoo%3Dbar', progress_listener=self.progress_listener) self._verify('Test File 3') def test_file_content_4(self): self.download_file_by_name( 'test.txt%253Ffoo%253Dbar', progress_listener=self.progress_listener ) self._verify('Test File 4') def test_file_info_1(self): download_version = self.bucket.get_file_info_by_name('test.txt?foo=bar') assert download_version.file_name == 'test.txt?foo=bar' assert download_version.file_info['custom_info'] == 'aaa?bbb' def test_file_info_2(self): download_version = self.bucket.get_file_info_by_name('test.txt%3Ffoo=bar') assert download_version.file_name == 'test.txt%3Ffoo=bar' assert download_version.file_info['custom_info'] == 'aaa%3Fbbb' def test_file_info_3(self): download_version = self.bucket.get_file_info_by_name('test.txt%3Ffoo%3Dbar') assert download_version.file_name == 'test.txt%3Ffoo%3Dbar' def test_file_info_4(self): download_version = self.bucket.get_file_info_by_name('test.txt%253Ffoo%253Dbar') assert download_version.file_name == 'test.txt%253Ffoo%253Dbar' class TestAuthorizeForBucket(TestCaseWithBucket): CACHE_CLASS = InMemoryCache @pytest.mark.apiver(from_ver=2) def test_authorize_for_bucket_ensures_cache(self): key = create_key( self.api, key_name='singlebucket', capabilities=[ 'listBuckets', ], bucket_id=self.bucket_id, ) self.api.authorize_account( application_key_id=key.id_, application_key=key.application_key, realm='production', ) # Check whether the bucket fetching performs an API call. with mock.patch.object(self.api, 'list_buckets') as mock_list_buckets: self.api.get_bucket_by_id(self.bucket_id) mock_list_buckets.assert_not_called() self.api.get_bucket_by_name(self.bucket_name) mock_list_buckets.assert_not_called() @pytest.mark.apiver(from_ver=2) def test_authorize_for_non_existing_bucket(self): key = create_key( self.api, key_name='singlebucket', capabilities=[ 'listBuckets', ], bucket_id=self.bucket_id + 'x', ) with self.assertRaises(RestrictedBucketMissing): self.api.authorize_account( application_key_id=key.id_, application_key=key.application_key, realm='production', ) class TestDownloadLocalDirectoryIssues(TestCaseWithBucket): def setUp(self): super().setUp() self.file_version = self.bucket.upload_bytes(b'test-data', 'file1') self.bytes_io = io.BytesIO() self.progress_listener = StubProgressListener() @pytest.mark.apiver(from_ver=2) def test_download_file_to_unknown_directory(self): with tempfile.TemporaryDirectory() as temp_dir: target_file = pathlib.Path(temp_dir) / 'non-existing-directory' / 'some-file' with self.assertRaises(DestinationDirectoryDoesntExist): self.bucket.download_file_by_name(self.file_version.file_name).save_to(target_file) @pytest.mark.apiver(from_ver=2) def test_download_file_targeting_directory(self): with tempfile.TemporaryDirectory() as temp_dir: target_file = pathlib.Path(temp_dir) / 'existing-directory' os.makedirs(target_file, exist_ok=True) with self.assertRaises(DestinationIsADirectory): self.bucket.download_file_by_name(self.file_version.file_name).save_to(target_file) @pytest.mark.apiver(from_ver=2) def test_download_file_targeting_directory_is_a_file(self): with tempfile.TemporaryDirectory() as temp_dir: some_file = pathlib.Path(temp_dir) / 'existing-file' some_file.write_bytes(b'i-am-a-file') target_file = some_file / 'save-target' with self.assertRaises(DestinationParentIsNotADirectory): self.bucket.download_file_by_name(self.file_version.file_name).save_to(target_file) @pytest.mark.apiver(from_ver=2) @pytest.mark.skipif( platform.system() == 'Windows', reason='os.chmod on Windows only affects read-only flag for files', ) def test_download_file_no_access_to_directory(self): chain = contextlib.ExitStack() temp_dir = chain.enter_context(tempfile.TemporaryDirectory()) with chain: target_directory = pathlib.Path(temp_dir) / 'impossible-directory' os.makedirs(target_directory, exist_ok=True) # Don't allow any operation on this directory. Used explicitly, as the documentation # states that on some platforms passing mode to `makedirs` may be ignored. os.chmod(target_directory, mode=0) # Ensuring that whenever we exit this context, our directory will be removable. chain.push(lambda *args, **kwargs: os.chmod(target_directory, mode=0o777)) target_file = target_directory / 'target_file' with self.assertRaises(DestinationDirectoryDoesntAllowOperation): self.bucket.download_file_by_name(self.file_version.file_name).save_to(target_file) class TestFileInfoB2Fields(TestCaseWithBucket): @dataclasses.dataclass class TestCase: fields: dict[str, str] expires_dt: datetime.datetime | None = None expires_parsed: datetime.datetime | None = None expires_parsed_raises: bool = False @property def kwargs(self) -> dict[str, str | datetime.datetime]: kws = {**self.fields} if self.expires_dt: kws['expires'] = self.expires_dt return kws def assert_(self, version): for name, value in self.fields.items(): assert getattr(version, name) == value if self.expires_parsed_raises: with pytest.raises(ValueError): version.expires_parsed() else: assert version.expires_parsed() == self.expires_parsed test_cases = [ TestCase(fields={}), TestCase( fields={ 'cache_control': 'max-age=3600', 'expires': 'Sun, 06 Nov 1994 08:49:37 GMT', 'content_disposition': 'attachment; filename="fname.ext"', 'content_encoding': 'utf-8', 'content_language': 'en_US', }, expires_parsed=datetime.datetime(1994, 11, 6, 8, 49, 37, tzinfo=datetime.timezone.utc), ), # RFC 850 format TestCase( fields={'expires': 'Sunday, 06-Nov-95 08:49:37 GMT'}, expires_parsed=datetime.datetime(1995, 11, 6, 8, 49, 37, tzinfo=datetime.timezone.utc), ), # ANSI C's asctime() format TestCase( fields={'expires': 'Sun Nov 6 08:49:37 1996'}, expires_parsed=datetime.datetime(1996, 11, 6, 8, 49, 37, tzinfo=datetime.timezone.utc), ), # Non-standard date format TestCase( fields={'expires': '2020-01-01 00:00:00'}, expires_parsed_raises=True, ), # Non-GMT timezone TestCase( fields={'expires': 'Sunday, 06-Nov-95 08:49:37 PDT'}, expires_parsed_raises=True, ), # Passing `expires`` as a datetime TestCase( fields={'expires': 'Sun, 06 Nov 1994 08:49:37 GMT'}, expires_dt=datetime.datetime(1994, 11, 6, 8, 49, 37, tzinfo=datetime.timezone.utc), expires_parsed=datetime.datetime(1994, 11, 6, 8, 49, 37, tzinfo=datetime.timezone.utc), expires_parsed_raises=False, ), # Passing `expires`` as a datetime in non-UTC timezone TestCase( fields={'expires': 'Sun, 06 Nov 1994 08:49:37 GMT'}, expires_dt=datetime.datetime( 1994, 11, 6, 9, 49, 37, tzinfo=datetime.timezone(datetime.timedelta(hours=1)) ), expires_parsed=datetime.datetime(1994, 11, 6, 8, 49, 37, tzinfo=datetime.timezone.utc), expires_parsed_raises=False, ), ] def test_upload_bytes(self): for test_case in self.test_cases: file_version = self.bucket.upload_bytes(b'', 'file1', **test_case.kwargs) test_case.assert_(file_version) def test_upload_unbound_stream(self): for test_case in self.test_cases: file_version = self.bucket.upload_unbound_stream( io.BytesIO(b'data'), 'file1', **test_case.kwargs ) test_case.assert_(file_version) def test_upload_empty_unbound_stream(self): for test_case in self.test_cases: file_version = self.bucket.upload_unbound_stream( io.BytesIO(b''), 'file1', **test_case.kwargs ) test_case.assert_(file_version) def test_upload_local_file(self): for test_case in self.test_cases: with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'' write_file(path, data) file_version = self.bucket.upload_local_file(path, 'file1', **test_case.kwargs) test_case.assert_(file_version) def test_copy_with_file_info(self): for test_case in self.test_cases: file_version = self.bucket.upload_bytes(b'', 'file1', **test_case.kwargs) copied_file_version = self.bucket.copy(file_version.id_, 'file2') test_case.assert_(copied_file_version) def test_copy_overwriting_file_info(self): for test_case in self.test_cases: file_version = self.bucket.upload_bytes(b'', 'file1') # For reasons I don't know, the content type must be supplied if and only if # file info is supplied - otherwise the SDK raises an exception forbidding that. content_type = None if test_case.fields: content_type = 'text/plain' copied_file_version = self.bucket.copy( file_version.id_, 'file2', content_type=content_type, **test_case.kwargs ) test_case.assert_(copied_file_version) def test_download_version(self): for test_case in self.test_cases: file_version = self.bucket.upload_bytes(b'', 'file1', **test_case.kwargs) download_file = self.bucket.download_file_by_id(file_version.id_) test_case.assert_(download_file.download_version) # Listing where every other response returns no entries and pointer to the next file class EmptyListBucketSimulator(BucketSimulator): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Whenever we receive a list request, if it's the first time # for this particular ``start_file_name``, we'll return # an empty response pointing to the same file. self.last_queried_file = None def _should_return_empty(self, file_name: str) -> bool: # Note that every other request is empty – the logic is as follows: # 1st request – unknown start name – empty response # 2nd request – known start name – normal response with a proper next filename # 3rd request – unknown start name (as it's the next filename from the previous request) – empty response # 4th request – known start name # etc. This works especially well when using limiter of number of files fetched set to 1. should_return_empty = self.last_queried_file != file_name self.last_queried_file = file_name return should_return_empty def list_file_versions( self, account_auth_token, start_file_name=None, start_file_id=None, max_file_count=None, # noqa prefix=None, ): if self._should_return_empty(start_file_name): return dict(files=[], nextFileName=start_file_name, nextFileId=start_file_id) return super().list_file_versions( account_auth_token, start_file_name, start_file_id, 1, # Forcing only a single file per response. prefix, ) def list_file_names( self, account_auth_token, start_file_name=None, max_file_count=None, # noqa prefix=None, ): if self._should_return_empty(start_file_name): return dict(files=[], nextFileName=start_file_name) return super().list_file_names( account_auth_token, start_file_name, 1, # Forcing only a single file per response. prefix, ) class EmptyListSimulator(RawSimulator): BUCKET_SIMULATOR_CLASS = EmptyListBucketSimulator class TestEmptyListVersions(TestListVersions): RAW_SIMULATOR_CLASS = EmptyListSimulator class TestEmptyLs(TestLs): RAW_SIMULATOR_CLASS = EmptyListSimulator def test_bucket_notification_rules(bucket, b2api_simulator): assert bucket.get_notification_rules() == [] notification_rule = { 'eventTypes': ['b2:ObjectCreated:*'], 'isEnabled': True, 'name': 'test-rule', 'objectNamePrefix': '', 'targetConfiguration': { 'customHeaders': [], 'targetType': 'webhook', 'url': 'https://example.com/webhook', }, } set_notification_rules = bucket.set_notification_rules([notification_rule]) assert set_notification_rules == bucket.get_notification_rules() assert_dict_equal_ignore_extra( set_notification_rules, [{**notification_rule, 'isSuspended': False, 'suspensionReason': ''}], ) b2api_simulator.bucket_id_to_bucket[bucket.id_].simulate_notification_rule_suspension( notification_rule['name'], 'simulated suspension' ) assert_dict_equal_ignore_extra( bucket.get_notification_rules(), [{**notification_rule, 'isSuspended': True, 'suspensionReason': 'simulated suspension'}], ) assert bucket.set_notification_rules([]) == [] assert bucket.get_notification_rules() == [] b2-sdk-python-2.8.0/test/unit/conftest.py000066400000000000000000000173571474454370000203040ustar00rootroot00000000000000###################################################################### # # File: test/unit/conftest.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import os import shutil import sys from glob import glob from pathlib import Path try: import ntsecuritycon import win32api import win32security except ImportError: ntsecuritycon = win32api = win32security = None import pytest pytest.register_assert_rewrite('test.unit') def get_api_versions(): return [ str(Path(p).parent.name) for p in sorted(glob(str(Path(__file__).parent / 'v*/apiver/'))) ] API_VERSIONS = get_api_versions() @pytest.hookimpl def pytest_addoption(parser): """Add an argument for running test for given apiver.""" parser.addoption( '--api', default=API_VERSIONS[-1], choices=API_VERSIONS, help='version of the API', ) @pytest.hookimpl def pytest_configure(config): """Add apiver test folder to the path and add "apiver" marker used by `pytest_runtest_setup`.""" sys.path.insert(0, str(Path(__file__).parent / config.getoption('--api') / 'apiver')) config.addinivalue_line( 'markers', 'apiver(*args, *, from_ver=0, to_ver=sys.maxsize): mark test to run only for specific apivers', ) @pytest.hookimpl def pytest_report_header(config): """Print apiver in the header.""" return 'b2sdk apiver: %s' % config.getoption('--api') @pytest.hookimpl(tryfirst=True) def pytest_ignore_collect(collection_path, config): """Ignore all tests from subfolders for different apiver.""" ver = config.getoption('--api') other_versions = [v for v in API_VERSIONS if v != ver] for other_version in other_versions: if other_version in collection_path.parts: return True return False def pytest_runtest_setup(item): """ Skip tests based on "apiver" marker. .. code-block:: python @pytest.mark.apiver(1) def test_only_for_v1(self): ... @pytest.mark.apiver(1, 3) def test_only_for_v1_and_v3(self): ... @pytest.mark.apiver(from_ver=2) def test_for_greater_or_equal_v2(self): ... @pytest.mark.apiver(to_ver=2) def test_for_less_or_equal_v2(self): ... @pytest.mark.apiver(from_ver=2, to_ver=4) def test_for_versions_from_v2_to_v4(self): ... Both `from_ver` and `to_ver` are inclusive. Providing test parameters based on apiver is also possible: .. code-block:: python @pytest.mark.parametrize( 'exc_class,exc_msg', [ pytest.param(InvalidArgument, None, marks=pytest.mark.apiver(from_ver=2)), pytest.param(re.error, "invalid something", marks=pytest.mark.apiver(to_ver=1)), ], ) def test_illegal_regex(self, exc_class, exc_msg): with pytest.raises(exc_class, match=exc_msg): error_raising_function() If a test is merked with "apiver" multiple times, it will be skipped if at least one of the apiver conditions cause it to be skipped. E.g. if a test module is marked with .. code-block:: python pytestmark = [pytest.mark.apiver(from_ver=1)] and a test function is marked with .. code-block:: python @pytest.mark.apiver(to_ver=1) def test_function(self): ... the test function will be run only for apiver=v1 """ for mark in item.iter_markers(name='apiver'): if mark.args and mark.kwargs: raise pytest.UsageError('apiver mark should not have both args and kwargs') int_ver = int(item.config.getoption('--api')[1:]) if mark.args: if int_ver not in mark.args: pytest.skip('test requires apiver to be one of: %s' % mark.args) elif mark.kwargs: from_ver = mark.kwargs.get('from_ver', 0) to_ver = mark.kwargs.get('to_ver', sys.maxsize) if not (from_ver <= int_ver <= to_ver): pytest.skip('test requires apiver to be in range: [%d, %d]' % (from_ver, to_ver)) @pytest.fixture(scope='session') def apiver(request): """Get apiver as a v-prefixed string, e.g. "v2".""" return request.config.getoption('--api') @pytest.fixture(scope='session') def apiver_int(apiver): """Get apiver as an int, e.g. `2`.""" return int(apiver[1:]) @pytest.fixture def b2api(): from apiver_deps import ( B2Api, B2HttpApiConfig, RawSimulator, StubAccountInfo, ) account_info = StubAccountInfo() api = B2Api( account_info, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator), ) simulator = api.session.raw_api account_id, master_key = simulator.create_account() api.authorize_account( application_key_id=account_id, application_key=master_key, realm='production', ) return api @pytest.fixture def b2api_simulator(b2api): return b2api.session.raw_api @pytest.fixture def bucket(b2api): return b2api.create_bucket('test-bucket', 'allPublic') @pytest.fixture def file_info(): return {'key': 'value'} class PermTool: def allow_access(self, path): pass def deny_access(self, path): pass class UnixPermTool(PermTool): def allow_access(self, path): path.chmod(0o700) def deny_access(self, path): path.chmod(0o000) class WindowsPermTool(PermTool): def __init__(self): self.user_sid = win32security.GetTokenInformation( win32security.OpenProcessToken(win32api.GetCurrentProcess(), win32security.TOKEN_QUERY), win32security.TokenUser, )[0] def allow_access(self, path): dacl = win32security.ACL() dacl.AddAccessAllowedAce( win32security.ACL_REVISION, ntsecuritycon.FILE_ALL_ACCESS, self.user_sid ) security_desc = win32security.GetFileSecurity( str(path), win32security.DACL_SECURITY_INFORMATION ) security_desc.SetSecurityDescriptorDacl(1, dacl, 0) win32security.SetFileSecurity( str(path), win32security.DACL_SECURITY_INFORMATION, security_desc ) def deny_access(self, path): dacl = win32security.ACL() dacl.AddAccessDeniedAce( win32security.ACL_REVISION, ntsecuritycon.FILE_ALL_ACCESS, self.user_sid ) security_desc = win32security.GetFileSecurity( str(path), win32security.DACL_SECURITY_INFORMATION ) security_desc.SetSecurityDescriptorDacl(1, dacl, 0) win32security.SetFileSecurity( str(path), win32security.DACL_SECURITY_INFORMATION, security_desc ) @pytest.fixture def fs_perm_tool(tmp_path): """ Ensure tmp_path is delete-able after the test. Important for the tests that mess with filesystem permissions. """ if os.name == 'nt': if win32api is None: pytest.skip('pywin32 is required to run this test') perm_tool = WindowsPermTool() else: perm_tool = UnixPermTool() yield perm_tool try: shutil.rmtree(tmp_path) except OSError: perm_tool.allow_access(tmp_path) for root, dirs, files in os.walk(tmp_path, topdown=True): for name in dirs: perm_tool.allow_access(Path(root) / name) for name in files: file_path = Path(root) / name perm_tool.allow_access(file_path) file_path.unlink() for root, dirs, files in os.walk(tmp_path, topdown=False): for name in dirs: (Path(root) / name).rmdir() tmp_path.rmdir() b2-sdk-python-2.8.0/test/unit/file_version/000077500000000000000000000000001474454370000205545ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/file_version/__init__.py000066400000000000000000000005201474454370000226620ustar00rootroot00000000000000###################################################################### # # File: test/unit/file_version/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/file_version/test_file_version.py000066400000000000000000000201171474454370000246520ustar00rootroot00000000000000###################################################################### # # File: test/unit/file_version/test_file_version.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import time import apiver_deps import pytest from apiver_deps import ( B2Api, B2HttpApiConfig, DownloadVersion, DummyCache, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, FileIdAndName, FileRetentionSetting, InMemoryAccountInfo, LegalHold, RawSimulator, RetentionMode, ) from apiver_deps_exception import AccessDenied, FileNotPresent if apiver_deps.V <= 1: from apiver_deps import FileVersionInfo as VFileVersion else: from apiver_deps import FileVersion as VFileVersion class TestFileVersion: @pytest.fixture(autouse=True) def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = DummyCache() self.api = B2Api( self.account_info, self.cache, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator) ) self.raw_api = self.api.session.raw_api (self.application_key_id, self.master_key) = self.raw_api.create_account() self.api.authorize_account( application_key_id=self.application_key_id, application_key=self.master_key, realm='production', ) self.bucket = self.api.create_bucket('testbucket', 'allPrivate', is_file_lock_enabled=True) self.file_version = self.bucket.upload_bytes( b'nothing', 'test_file', cache_control='private, max-age=3600' ) @pytest.mark.apiver(to_ver=1) def test_format_ls_entry(self): file_version_info = VFileVersion( 'a2', 'inner/a.txt', 200, 'text/plain', 'sha1', {}, 2000, 'upload' ) expected_entry = ( ' ' ' a2 upload 1970-01-01 ' '00:00:02 200 inner/a.txt' ) assert expected_entry == file_version_info.format_ls_entry() def test_get_fresh_state(self): self.api.update_file_legal_hold( self.file_version.id_, self.file_version.file_name, LegalHold.ON ) fetched_version = self.api.get_file_info(self.file_version.id_) if apiver_deps.V <= 1: fetched_version = self.api.file_version_factory.from_api_response(fetched_version) assert self.file_version.as_dict() != fetched_version.as_dict() refreshed_version = self.file_version.get_fresh_state() assert isinstance(refreshed_version, VFileVersion) assert refreshed_version.as_dict() == fetched_version.as_dict() def test_clone_file_version_and_download_version(self): encryption = EncryptionSetting( EncryptionMode.SSE_C, EncryptionAlgorithm.AES256, EncryptionKey(b'secret', None) ) initial_file_version = self.bucket.upload_bytes( b'nothing', 'test_file', content_type='video/mp4', file_info={ 'file': 'info', 'b2-content-language': 'en_US', 'b2-content-disposition': 'attachment', 'b2-expires': '2100-01-01', 'b2-content-encoding': 'text', }, encryption=encryption, file_retention=FileRetentionSetting(RetentionMode.GOVERNANCE, 100), legal_hold=LegalHold.ON, cache_control='public, max-age=86400', ) assert initial_file_version._clone() == initial_file_version cloned = initial_file_version._clone(legal_hold=LegalHold.OFF) assert isinstance(cloned, VFileVersion) assert cloned.as_dict() == { **initial_file_version.as_dict(), 'legalHold': LegalHold.OFF.value, } download_version = self.api.download_file_by_id( initial_file_version.id_, encryption=encryption ).download_version assert download_version._clone() == download_version cloned = download_version._clone(legal_hold=LegalHold.OFF) assert isinstance(cloned, DownloadVersion) assert cloned.as_dict() == {**download_version.as_dict(), 'legalHold': LegalHold.OFF.value} def test_update_legal_hold(self): new_file_version = self.file_version.update_legal_hold(LegalHold.ON) assert isinstance(new_file_version, VFileVersion) assert new_file_version.legal_hold == LegalHold.ON download_version = self.api.download_file_by_id(self.file_version.id_).download_version new_download_version = download_version.update_legal_hold(LegalHold.ON) assert isinstance(new_download_version, DownloadVersion) assert new_download_version.legal_hold == LegalHold.ON def test_update_retention(self): new_retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 100) new_file_version = self.file_version.update_retention(new_retention) assert isinstance(new_file_version, VFileVersion) assert new_file_version.file_retention == new_retention download_version = self.api.download_file_by_id(self.file_version.id_).download_version new_download_version = download_version.update_retention(new_retention) assert isinstance(new_download_version, DownloadVersion) assert new_download_version.file_retention == new_retention def test_delete_file_version(self): ret = self.file_version.delete() assert isinstance(ret, FileIdAndName) with pytest.raises(FileNotPresent): self.bucket.get_file_info_by_name(self.file_version.file_name) def test_delete_bypass_governance(self): locked_file_version = self.bucket.upload_bytes( b'nothing', 'test_file_with_governance', file_retention=FileRetentionSetting(RetentionMode.GOVERNANCE, int(time.time()) + 100), ) with pytest.raises(AccessDenied): locked_file_version.delete() locked_file_version.delete(bypass_governance=True) with pytest.raises(FileNotPresent): self.bucket.get_file_info_by_name(locked_file_version.file_name) def test_delete_download_version(self): download_version = self.api.download_file_by_id(self.file_version.id_).download_version ret = download_version.delete() assert isinstance(ret, FileIdAndName) with pytest.raises(FileNotPresent): self.bucket.get_file_info_by_name(self.file_version.file_name) def test_file_version_upload_headers(self): file_version = self.file_version._clone( server_side_encryption=EncryptionSetting( EncryptionMode.SSE_C, EncryptionAlgorithm.AES256, EncryptionKey(None, None), ), ) assert ( file_version._get_upload_headers() == """ Authorization: auth_token_0 Content-Length: 7 X-Bz-File-Name: test_file Content-Type: b2/x-auto X-Bz-Content-Sha1: 0feca720e2c29dafb2c900713ba560e03b758711 X-Bz-Info-b2-cache-control: private%2C%20max-age%3D3600 X-Bz-Server-Side-Encryption-Customer-Algorithm: AES256 X-Bz-Server-Side-Encryption-Customer-Key: KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKio= X-Bz-Server-Side-Encryption-Customer-Key-Md5: SaaDheEjzuynJH8eW6AEpQ== X-Bz-File-Legal-Hold: off X-Bz-File-Retention-Mode: None X-Bz-File-Retention-Retain-Until-Timestamp: None """.strip() .replace(': ', '') .replace(' ', '') .replace('\n', '') .encode('utf8') ) assert not file_version.has_large_header file_version.file_info['dummy'] = 'a' * 2000 # make metadata > 2k bytes assert file_version.has_large_header # FileVersion.download tests are not here, because another test file already has all the facilities for such test # prepared b2-sdk-python-2.8.0/test/unit/filter/000077500000000000000000000000001474454370000173555ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/filter/__init__.py000066400000000000000000000005121474454370000214640ustar00rootroot00000000000000###################################################################### # # File: test/unit/filter/__init__.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/filter/test_filter.py000066400000000000000000000025361474454370000222610ustar00rootroot00000000000000###################################################################### # # File: test/unit/filter/test_filter.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from apiver_deps import Filter from b2sdk._internal.filter import FilterMatcher @pytest.mark.parametrize( ('filters', 'expr', 'expected'), ( ([], 'a', True), ([Filter.exclude('*')], 'something', False), ([Filter.include('a-*')], 'a-', True), ([Filter.include('a-*')], 'b-', False), ([Filter.exclude('*.txt')], 'a.txt', False), ([Filter.exclude('*.txt')], 'a.csv', True), ([Filter.exclude('*'), Filter.include('*.[ct]sv')], 'a.csv', True), ([Filter.exclude('*'), Filter.include('*.[ct]sv')], 'a.tsv', True), ([Filter.exclude('*'), Filter.include('*.[ct]sv')], 'a.ksv', False), ( [Filter.exclude('*'), Filter.include('*.[ct]sv'), Filter.exclude('a.csv')], 'a.csv', False, ), ([Filter.exclude('*'), Filter.include('*.[ct]sv'), Filter.exclude('a.csv')], 'b.csv', True), ), ) def test_filter_matcher(filters, expr, expected): assert FilterMatcher(filters).match(expr) == expected b2-sdk-python-2.8.0/test/unit/fixtures/000077500000000000000000000000001474454370000177415ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/fixtures/__init__.py000066400000000000000000000006461474454370000220600ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from .b2http import * from .cache import * from .raw_api import * from .session import * b2-sdk-python-2.8.0/test/unit/fixtures/b2http.py000066400000000000000000000007371474454370000215250ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/b2http.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from apiver_deps import B2Http @pytest.fixture def fake_b2http(mocker): return mocker.MagicMock(name='FakeB2Http', spec=B2Http) b2-sdk-python-2.8.0/test/unit/fixtures/cache.py000066400000000000000000000007521474454370000213620ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/cache.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from apiver_deps import InMemoryCache @pytest.fixture def fake_cache(mocker): return mocker.MagicMock(name='FakeCache', spec=InMemoryCache) b2-sdk-python-2.8.0/test/unit/fixtures/folder.py000066400000000000000000000062611474454370000215730ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/folder.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from unittest import mock import apiver_deps import pytest from apiver_deps import DEFAULT_SCAN_MANAGER, B2Folder, LocalFolder, LocalPath if apiver_deps.V <= 1: from apiver_deps import FileVersionInfo as VFileVersion else: from apiver_deps import FileVersion as VFileVersion class FakeB2Folder(B2Folder): def __init__(self, test_files): self.file_versions = [] for test_file in test_files: self.file_versions.extend(self._file_versions(*test_file)) super().__init__('test-bucket', 'folder', mock.MagicMock()) def get_file_versions(self): yield from sorted(self.file_versions, key=lambda x: x.file_name) def _file_versions(self, name, mod_times, size=10): """ Makes FileVersion objects. Positive modification times are uploads, and negative modification times are hides. It's a hack, but it works. """ if apiver_deps.V <= 1: mandatory_kwargs = {} else: mandatory_kwargs = { 'api': None, 'account_id': 'account-id', 'bucket_id': 'bucket-id', 'content_md5': 'content_md5', 'server_side_encryption': None, } return [ VFileVersion( id_='id_%s_%d' % (name[0], abs(mod_time)), file_name='folder/' + name, upload_timestamp=abs(mod_time), action='upload' if 0 < mod_time else 'hide', size=size, file_info={'in_b2': 'yes'}, content_type='text/plain', content_sha1='content_sha1', **mandatory_kwargs, ) for mod_time in mod_times ] class FakeLocalFolder(LocalFolder): def __init__(self, test_files): super().__init__('folder') self.local_paths = [self._local_path(*test_file) for test_file in test_files] def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): for single_path in sorted(self.local_paths, key=lambda x: x.relative_path): if single_path.relative_path.endswith('/'): if policies_manager.should_exclude_b2_directory(single_path.relative_path): continue else: if policies_manager.should_exclude_local_path(single_path): continue yield single_path def make_full_path(self, name): return '/dir/' + name def _local_path(self, name, mod_times, size=10): """ Makes a LocalPath object for a local file. """ return LocalPath(name, name, mod_times[0], size) @pytest.fixture(scope='session') def folder_factory(): def get_folder(f_type, *files): if f_type == 'b2': return FakeB2Folder(files) return FakeLocalFolder(files) return get_folder b2-sdk-python-2.8.0/test/unit/fixtures/raw_api.py000066400000000000000000000031361474454370000217400ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/raw_api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from copy import copy import pytest from apiver_deps import ALL_CAPABILITIES, B2RawHTTPApi @pytest.fixture def fake_b2_raw_api_responses(): return { 'authorize_account': { 'accountId': '6012deadbeef', 'apiInfo': { 'groupsApi': {}, 'storageApi': { 'bucketId': None, 'bucketName': None, 'capabilities': copy(ALL_CAPABILITIES), 'namePrefix': None, 'downloadUrl': 'https://f000.backblazeb2.xyz:8180', 'absoluteMinimumPartSize': 5000000, 'recommendedPartSize': 100000000, 'apiUrl': 'https://api000.backblazeb2.xyz:8180', 's3ApiUrl': 'https://s3.us-west-000.backblazeb2.xyz:8180', }, }, 'authorizationToken': '4_1111111111111111111111111_11111111_111111_1111_1111111111111_1111_11111111=', } } @pytest.fixture def fake_b2_raw_api(mocker, fake_b2http, fake_b2_raw_api_responses): raw_api = mocker.MagicMock(name='FakeB2RawHTTPApi', spec=B2RawHTTPApi) raw_api.b2_http = fake_b2http raw_api.authorize_account.return_value = fake_b2_raw_api_responses['authorize_account'] return raw_api b2-sdk-python-2.8.0/test/unit/fixtures/session.py000066400000000000000000000016111474454370000217750ustar00rootroot00000000000000###################################################################### # # File: test/unit/fixtures/session.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from apiver_deps import B2Session @pytest.fixture def b2_session(fake_account_info, fake_cache, fake_b2_raw_api): session = B2Session(account_info=fake_account_info, cache=fake_cache) session.raw_api = fake_b2_raw_api return session @pytest.fixture def fake_b2_session(mocker, fake_account_info, fake_cache, fake_b2_raw_api): b2_session = mocker.MagicMock(name='FakeB2Session', spec=B2Session) b2_session.account_info = fake_account_info b2_session.cache = fake_cache b2_session.raw_api = fake_b2_raw_api return b2_session b2-sdk-python-2.8.0/test/unit/internal/000077500000000000000000000000001474454370000177045ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/internal/__init__.py000066400000000000000000000005141474454370000220150ustar00rootroot00000000000000###################################################################### # # File: test/unit/internal/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/internal/test_emerge_planner.py000066400000000000000000000627661474454370000243210ustar00rootroot00000000000000###################################################################### # # File: test/unit/internal/test_emerge_planner.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from unittest.mock import create_autospec import pytest from b2sdk._internal.account_info.abstract import AbstractAccountInfo from b2sdk._internal.http_constants import ( DEFAULT_MAX_PART_SIZE, DEFAULT_MIN_PART_SIZE, DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE, GIGABYTE, MEGABYTE, ) from b2sdk._internal.transfer.emerge.planner.part_definition import ( CopyEmergePartDefinition, UploadEmergePartDefinition, UploadSubpartsEmergePartDefinition, ) from b2sdk._internal.transfer.emerge.planner.planner import EmergePlanner from b2sdk._internal.transfer.emerge.planner.upload_subpart import ( LocalSourceUploadSubpart, RemoteSourceUploadSubpart, ) from b2sdk._internal.transfer.emerge.write_intent import WriteIntent from b2sdk._internal.transfer.outbound.copy_source import CopySource as OrigCopySource from b2sdk._internal.transfer.outbound.upload_source import UploadSourceStream from ..test_base import TestBase class UploadSource(UploadSourceStream): def __init__(self, length): super().__init__(lambda: None, length) class CopySource(OrigCopySource): def __init__(self, length): super().__init__(id(self), length=length) def part(source_or_def_list, *offset_len): """Helper for building emerge parts from outbound sources defs. Makes planner tests easier to read. Possible "def" structures: ``outbound_source`` - will build correct emerge part using type introspection with ``0`` for ``offset`` and ``outbound_source.get_content_length()`` for ``length`` ``outbound_source, offset, length`` - same as above, but given ``offset`` and ``length`` ``[outbound_source_def,...]``` - will build emerge part using ``UploadSubpartsEmergePartDefinition`` and type introspection for upload subparts; ``outbound_source_def`` is similar to above, either outbound source or tuple with source, offset, length Please notice that ``part([copy_source])`` is a single "small copy" - first download then upload, and ``part(copy_source) is "regular copy" - ``length`` is not verified here """ if isinstance(source_or_def_list, list): assert not offset_len subparts = [] for sub_part_meta in source_or_def_list: if isinstance(sub_part_meta, tuple): source, offset, length = sub_part_meta else: source = sub_part_meta offset = 0 length = source.get_content_length() if isinstance(source, UploadSource): subparts.append(LocalSourceUploadSubpart(source, offset, length)) else: subparts.append(RemoteSourceUploadSubpart(source, offset, length)) return UploadSubpartsEmergePartDefinition(subparts) else: source = source_or_def_list if offset_len: offset, length = offset_len else: offset = 0 length = source.get_content_length() if isinstance(source, UploadSource): return UploadEmergePartDefinition(source, offset, length) else: return CopyEmergePartDefinition(source, offset, length) class TestEmergePlanner(TestBase): def setUp(self): # we want to hardcode here current production settings and change them # whenever these are changed on production, and relate all test lengths # to those sizes - if test assumes something about those values # (like `min_part_size` > 2 * MEGABYTE), then one should add assertion # at the beginning of test function body self.recommended_size = 100 * MEGABYTE self.min_size = 5 * MEGABYTE self.max_size = 5 * GIGABYTE self.planner = self._get_emerge_planner() def _get_emerge_planner(self): return EmergePlanner( min_part_size=self.min_size, recommended_upload_part_size=self.recommended_size, max_part_size=self.max_size, ) def test_part_sizes(self): self.assertGreater(self.min_size, 0) self.assertGreaterEqual(self.recommended_size, self.min_size) self.assertGreaterEqual(self.max_size, self.recommended_size) def test_simple_concatenate(self): sources = [ CopySource(self.recommended_size), UploadSource(self.recommended_size), CopySource(self.recommended_size), UploadSource(self.recommended_size), ] self.verify_emerge_plan_for_write_intents( WriteIntent.wrap_sources_iterator(sources), [part(source) for source in sources], ) def test_single_part_upload(self): source = UploadSource(self.recommended_size) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], [part(source)], ) def test_single_part_copy(self): source = CopySource(self.max_size) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], [part(source)], ) def test_single_multipart_upload(self): self.assertGreater(self.recommended_size, 2 * self.min_size) remainder = 2 * self.min_size source = UploadSource(self.recommended_size * 5 + remainder) expected_part_sizes = [self.recommended_size] * 5 + [remainder] self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], self.split_source_to_part_defs(source, expected_part_sizes), ) def test_recommended_part_size_decrease(self): source_upload1 = UploadSource(self.recommended_size * 10001) write_intents = [ WriteIntent(source_upload1), ] emerge_plan = self.planner.get_emerge_plan(write_intents) assert len(emerge_plan.emerge_parts) < 10000 def test_single_multipart_copy(self): source = CopySource(5 * self.max_size) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], self.split_source_to_part_defs(source, [self.max_size] * 5) ) def test_single_multipart_copy_remainder(self): self.assertGreaterEqual(self.min_size, 2) source = CopySource(5 * self.max_size + int(self.min_size / 2)) expected_part_count = 7 base_part_size = int(source.get_content_length() / expected_part_count) size_remainder = source.get_content_length() % expected_part_count expected_part_sizes = [base_part_size + 1] * size_remainder + [base_part_size] * ( expected_part_count - size_remainder ) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], self.split_source_to_part_defs(source, expected_part_sizes), ) def test_single_small_copy(self): source = CopySource(self.min_size - 1) self.verify_emerge_plan_for_write_intents( [WriteIntent(source)], [ # single small copy should be processed using `copy_file` # which does not have minimum file size limit part(source), ], ) def test_copy_then_small_copy(self): source_copy = CopySource(self.recommended_size) source_small_copy = CopySource(self.min_size - 1) write_intents = WriteIntent.wrap_sources_iterator([source_copy, source_small_copy]) self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy), part([source_small_copy]), # this means: download and then upload ], ) def test_small_copy_then_copy(self): self.assertGreater(self.min_size, MEGABYTE) source_small_copy = CopySource(self.min_size - MEGABYTE) source_copy = CopySource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator([source_small_copy, source_copy]) self.verify_emerge_plan_for_write_intents( write_intents, [ part( [ source_small_copy, (source_copy, 0, MEGABYTE), ] ), part(source_copy, MEGABYTE, self.recommended_size - MEGABYTE), ], ) def test_upload_small_copy_then_copy(self): source_upload = UploadSource(self.recommended_size) source_small_copy = CopySource(self.min_size - 1) source_copy = CopySource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator( [source_upload, source_small_copy, source_copy] ) self.verify_emerge_plan_for_write_intents( write_intents, [ part( [ source_upload, source_small_copy, ] ), part(source_copy), ], ) def test_upload_small_copy_x2_then_copy(self): source_upload = UploadSource(self.recommended_size) source_small_copy1 = CopySource(length=self.min_size - 1) source_small_copy2 = CopySource(length=self.min_size - 1) source_copy = CopySource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator( [source_upload, source_small_copy1, source_small_copy2, source_copy] ) self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_upload), part( [ source_small_copy1, source_small_copy2, ] ), part(source_copy), ], ) def test_upload_multiple_sources(self): self.assertEqual(self.recommended_size % 8, 0) unit_part_size = int(self.recommended_size / 8) uneven_part_size = 3 * unit_part_size sources = [UploadSource(uneven_part_size) for i in range(8)] self.verify_emerge_plan_for_write_intents( WriteIntent.wrap_sources_iterator(sources), [ part( [ sources[0], sources[1], (sources[2], 0, 2 * unit_part_size), ] ), part( [ (sources[2], 2 * unit_part_size, unit_part_size), sources[3], sources[4], (sources[5], 0, unit_part_size), ] ), part( [ (sources[5], unit_part_size, 2 * unit_part_size), sources[6], sources[7], ] ), ], ) def test_small_upload_not_enough_copy_then_upload(self): self.assertGreater(self.min_size, 2 * MEGABYTE) source_small_upload = UploadSource(self.min_size - 2 * MEGABYTE) source_copy = CopySource(self.min_size + MEGABYTE) source_upload = UploadSource(self.recommended_size) write_intents = WriteIntent.wrap_sources_iterator( [source_small_upload, source_copy, source_upload] ) small_parts_len = ( source_small_upload.get_content_length() + source_copy.get_content_length() ) source_upload_split_offset = self.recommended_size - small_parts_len self.verify_emerge_plan_for_write_intents( write_intents, [ part( [ source_small_upload, source_copy, (source_upload, 0, source_upload_split_offset), ] ), part(source_upload, source_upload_split_offset, small_parts_len), ], ) def test_basic_local_overlap(self): source1 = UploadSource(self.recommended_size * 2) source2 = UploadSource(self.recommended_size * 2) write_intents = [ WriteIntent(source1), WriteIntent(source2, destination_offset=self.recommended_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [part(source1, 0, self.recommended_size)] + self.split_source_to_part_defs(source2, [self.recommended_size] * 2), ) def test_local_stairs_overlap(self): """ intent 0 #### intent 1 #### intent 2 #### intent 3 #### """ self.assertEqual(self.recommended_size % 4, 0) shift = int(self.recommended_size / 4) sources = [UploadSource(self.recommended_size) for i in range(4)] write_intents = [ WriteIntent(source, destination_offset=i * shift) for i, source in enumerate(sources) ] three_quarters = int(3 * self.recommended_size / 4) # 1234567 # su1: **** # su2: XXXX # su3: XXXX # su4: X*** self.verify_emerge_plan_for_write_intents( write_intents, [ part([(sources[0], 0, three_quarters), (sources[-1], 0, shift)]), part(sources[-1], shift, three_quarters), ], ) def test_local_remote_overlap_start(self): source_upload = UploadSource(self.recommended_size * 2) source_copy = CopySource(self.recommended_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy), ] self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy), part(source_upload, self.recommended_size, self.recommended_size), ], ) def test_local_remote_overlap_end(self): source_upload = UploadSource(self.recommended_size * 2) source_copy = CopySource(self.recommended_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy, destination_offset=self.recommended_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_upload, 0, self.recommended_size), part(source_copy), ], ) def test_local_remote_overlap_middle(self): source_upload = UploadSource(self.recommended_size * 3) source_copy = CopySource(self.recommended_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy, destination_offset=self.recommended_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_upload, 0, self.recommended_size), part(source_copy), part(source_upload, 2 * self.recommended_size, self.recommended_size), ], ) def test_local_small_copy_overlap(self): self.assertGreater(self.recommended_size, self.min_size * 3 - 3) source_upload = UploadSource(self.recommended_size) small_size = self.min_size - 1 source_copy1 = CopySource(small_size) source_copy2 = CopySource(small_size) source_copy3 = CopySource(small_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=small_size), WriteIntent(source_copy3, destination_offset=2 * small_size), ] self.verify_emerge_plan_for_write_intents( write_intents, [part(source_upload)], ) def test_overlap_cause_small_copy_remainder_2_intent_case(self): self.assertGreater(self.min_size, 2 * MEGABYTE) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_copy1 = CopySource(copy_size) source_copy2 = CopySource(self.min_size) write_intents = [ WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), ] # 123456789 # sc1: ****** # sc2: XX*** self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, copy_size), part( [ ( source_copy2, 2 * MEGABYTE, self.min_size - 2 * MEGABYTE, ), # this means: download and then upload ] ), ], ) def test_overlap_cause_small_copy_remainder_3_intent_case(self): self.assertGreater(self.min_size, MEGABYTE) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_copy1 = CopySource(copy_size) source_copy2 = CopySource(copy_size) source_copy3 = CopySource(copy_size) write_intents = [ WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), WriteIntent(source_copy3, destination_offset=2 * copy_overlap_offset), ] # 12345678901234 # sc1: *****X # sc2: X***** # sc3: XX**** self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, self.min_size), part(source_copy2, MEGABYTE, self.min_size), part( [ ( source_copy3, 2 * MEGABYTE, copy_overlap_offset, ), # this means: download and then upload ] ), ], ) def test_overlap_protected_copy_and_upload(self): self.assertGreater(self.min_size, MEGABYTE) self.assertGreater(self.recommended_size, 2 * self.min_size) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_upload = UploadSource(self.recommended_size) source_copy1 = CopySource(copy_size) source_copy2 = CopySource(copy_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), ] upload_offset = copy_overlap_offset + copy_size # 123456789012 # su: XXXXXXXXXX**(...) # sc1: *****X # sc2: X***** self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, self.min_size), part(source_copy2, MEGABYTE, self.min_size), part(source_upload, upload_offset, self.recommended_size - upload_offset), ], ) def test_overlap_copy_and_small_copy_remainder_and_upload(self): self.assertGreater(self.min_size, 2 * MEGABYTE) self.assertGreater(self.recommended_size, self.min_size + MEGABYTE) copy_size = self.min_size + MEGABYTE copy_overlap_offset = copy_size - 2 * MEGABYTE source_upload = UploadSource(self.recommended_size) source_copy1 = CopySource(copy_size) source_copy2 = CopySource(self.min_size) write_intents = [ WriteIntent(source_upload), WriteIntent(source_copy1), WriteIntent(source_copy2, destination_offset=copy_overlap_offset), ] # 12345678901 # su: XXXXXX*****(...) # sc1: ****** # sc2: XXXXX self.verify_emerge_plan_for_write_intents( write_intents, [ part(source_copy1, 0, copy_size), part(source_upload, copy_size, self.recommended_size - copy_size), ], ) def test_raise_on_hole(self): source_upload1 = UploadSource(self.recommended_size) source_upload2 = UploadSource(self.recommended_size) source_copy1 = CopySource(self.recommended_size) source_copy2 = CopySource(self.recommended_size) write_intents = [ WriteIntent(source_upload1), WriteIntent(source_upload2, destination_offset=self.recommended_size + 2 * MEGABYTE), WriteIntent(source_copy1, destination_offset=MEGABYTE), WriteIntent(source_copy2, destination_offset=self.recommended_size + 3 * MEGABYTE), ] hole_msg = ( 'Cannot emerge file with holes. ' f'Found hole range: ({write_intents[2].destination_end_offset}, {write_intents[1].destination_offset})' ) with self.assertRaises(ValueError, hole_msg): self.planner.get_emerge_plan(write_intents) def test_empty_upload(self): source_upload = UploadSource(0) self.verify_emerge_plan_for_write_intents( [WriteIntent(source_upload)], [part(source_upload)], ) def verify_emerge_plan_for_write_intents(self, write_intents, expected_part_defs): emerge_plan = self.planner.get_emerge_plan(write_intents) self.assert_same_part_definitions(emerge_plan, expected_part_defs) def split_source_to_part_defs(self, source, part_sizes): if isinstance(source, UploadSource): def_class = UploadEmergePartDefinition else: def_class = CopyEmergePartDefinition expected_part_defs = [] current_offset = 0 for part_size in part_sizes: expected_part_defs.append(def_class(source, current_offset, part_size)) current_offset += part_size return expected_part_defs def assert_same_part_definitions(self, emerge_plan, expected_part_defs): emerge_parts = list(emerge_plan.emerge_parts) self.assertEqual(len(emerge_parts), len(expected_part_defs)) for emerge_part, expected_part_def in zip(emerge_parts, expected_part_defs): emerge_part_def = emerge_part.part_definition self.assertIs(emerge_part_def.__class__, expected_part_def.__class__) if isinstance(emerge_part_def, UploadSubpartsEmergePartDefinition): upload_subparts = emerge_part_def.upload_subparts expected_subparts = expected_part_def.upload_subparts self.assertEqual(len(upload_subparts), len(expected_subparts)) for subpart, expected_subpart in zip(upload_subparts, expected_subparts): self.assertIs(subpart.__class__, expected_subpart.__class__) self.assertIs(subpart.outbound_source, expected_subpart.outbound_source) self.assertEqual(subpart.relative_offset, expected_subpart.relative_offset) self.assertEqual(subpart.length, expected_subpart.length) else: if isinstance(emerge_part_def, UploadEmergePartDefinition): self.assertIs(emerge_part_def.upload_source, expected_part_def.upload_source) else: self.assertIs(emerge_part_def.copy_source, expected_part_def.copy_source) self.assertEqual(emerge_part_def.relative_offset, expected_part_def.relative_offset) self.assertEqual(emerge_part_def.length, expected_part_def.length) @pytest.fixture def account_info(): mock_account_info = create_autospec(AbstractAccountInfo) mock_account_info.get_recommended_part_size.return_value = 150 * MEGABYTE return mock_account_info def test_emerge_planner_from_account_info(account_info): planner = EmergePlanner.from_account_info(account_info=account_info) assert planner.min_part_size == DEFAULT_MIN_PART_SIZE assert planner.recommended_upload_part_size == 150 * MEGABYTE assert planner.max_part_size == DEFAULT_MAX_PART_SIZE @pytest.mark.parametrize( 'min_part_size, recommended_upload_part_size, max_part_size, expected', [ (100 * MEGABYTE, None, None, {'min_part_size': 100 * MEGABYTE}), ( GIGABYTE, GIGABYTE, None, {'min_part_size': GIGABYTE, 'recommended_upload_part_size': GIGABYTE}, ), ( None, None, DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE // 2, { 'recommended_upload_part_size': DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE // 2, 'max_part_size': DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE // 2, }, ), ], ) def test_emerge_planner_from_account_info__with_explicitly_set_params( min_part_size, recommended_upload_part_size, max_part_size, expected, account_info ): planner = EmergePlanner.from_account_info( account_info=account_info, min_part_size=min_part_size, recommended_upload_part_size=recommended_upload_part_size, max_part_size=max_part_size, ) assert planner.min_part_size == expected.get('min_part_size', DEFAULT_MIN_PART_SIZE) assert planner.recommended_upload_part_size == expected.get( 'recommended_upload_part_size', 150 * MEGABYTE ) assert planner.max_part_size == expected.get('max_part_size', DEFAULT_MAX_PART_SIZE) b2-sdk-python-2.8.0/test/unit/internal/test_unbound_write_intent.py000066400000000000000000000116711474454370000255700ustar00rootroot00000000000000###################################################################### # # File: test/unit/internal/test_unbound_write_intent.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import io import string from unittest.mock import MagicMock from b2sdk._internal.transfer.emerge.unbound_write_intent import ( IOWrapper, UnboundSourceBytes, UnboundStreamBufferTimeout, UnboundWriteIntentGenerator, ) from b2sdk._internal.transfer.emerge.write_intent import WriteIntent from b2sdk._internal.utils import hex_sha1_of_bytes from ..test_base import TestBase class TestIOWrapper(TestBase): def setUp(self) -> None: self.data = b'test-data' self.mock_fun = MagicMock() self.wrapper = IOWrapper(self.data, release_function=self.mock_fun) def test_function_called_on_close_manual(self): self.mock_fun.assert_not_called() self.wrapper.read(len(self.data)) self.mock_fun.assert_not_called() self.wrapper.read(len(self.data)) self.mock_fun.assert_not_called() self.wrapper.close() self.mock_fun.assert_called_once() def test_function_called_on_close_context(self): self.mock_fun.assert_not_called() with self.wrapper as w: w.read(len(self.data)) self.mock_fun.assert_called_once() class TestUnboundSourceBytes(TestBase): def test_data_has_length_and_sha1_calculated_without_touching_the_stream(self): data = bytearray(b'test-data') mock_fun = MagicMock() source = UnboundSourceBytes(data, mock_fun) self.assertEqual(len(data), source.get_content_length()) self.assertEqual(hex_sha1_of_bytes(data), source.get_content_sha1()) mock_fun.assert_not_called() class TestUnboundWriteIntentGenerator(TestBase): def setUp(self) -> None: self.data = b'test-data' self.kwargs = dict( # From the perspective of the UnboundWriteIntentGenerator itself, the queue size # can be any positive integer. Bucket requires it to be at least two, so that # it can determine the upload method. queue_size=1, queue_timeout_seconds=0.1, ) def _get_iterator(self, buffer_and_read_size: int = 1, data: bytes | None = None): data = data or self.data generator = UnboundWriteIntentGenerator( io.BytesIO(data), buffer_size_bytes=buffer_and_read_size, read_size=buffer_and_read_size, **self.kwargs, ) return generator.iterator() def _read_write_intent(self, write_intent: WriteIntent, full_read_size: int = 1) -> bytes: buffer_stream = write_intent.outbound_source.open() # noqa read_data = buffer_stream.read(full_read_size) empty_data = buffer_stream.read(full_read_size) self.assertEqual(0, len(empty_data)) buffer_stream.close() return read_data def test_timeout_called_when_waiting_too_long_for_empty_buffer_slot(self): # First buffer is delivered without issues. iterator = self._get_iterator() next(iterator) with self.assertRaises(UnboundStreamBufferTimeout): # Since we didn't read the first one, the second one is blocked. next(iterator) def test_all_data_iterated_over(self): # This also tests empty last buffer case. data_loaded = [] for write_intent in self._get_iterator(): read_data = self._read_write_intent(write_intent, 1) self.assertEqual( self.data[write_intent.destination_offset].to_bytes(1, 'big'), read_data, ) data_loaded.append((read_data, write_intent.destination_offset)) expected_data_loaded = [ (byte.to_bytes(1, 'big'), idx) for idx, byte in enumerate(self.data) ] self.assertCountEqual(expected_data_loaded, data_loaded) def test_larger_buffer_size(self): # This also tests non-empty last buffer case. read_size = 4 # Build a buffer of N reads of size read_size and one more byte. data = b''.join(string.printable[:read_size].encode('ascii') for _ in range(2)) + b'1' for write_intent in self._get_iterator(read_size, data): read_data = self._read_write_intent(write_intent, full_read_size=read_size) offset = write_intent.destination_offset expected_data = data[offset : offset + read_size] self.assertEqual(expected_data, read_data) def test_single_buffer_delivered(self): read_size = len(self.data) + 1 iterator = self._get_iterator(read_size) write_intent = next(iterator) self._read_write_intent(write_intent, full_read_size=read_size) with self.assertRaises(StopIteration): next(iterator) b2-sdk-python-2.8.0/test/unit/internal/transfer/000077500000000000000000000000001474454370000215305ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/internal/transfer/__init__.py000066400000000000000000000004621474454370000236430ustar00rootroot00000000000000###################################################################### # # File: test/unit/internal/transfer/__init__.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-2.8.0/test/unit/internal/transfer/downloader/000077500000000000000000000000001474454370000236665ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/internal/transfer/downloader/__init__.py000066400000000000000000000004751474454370000260050ustar00rootroot00000000000000###################################################################### # # File: test/unit/internal/transfer/downloader/__init__.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-2.8.0/test/unit/internal/transfer/downloader/test_parallel.py000066400000000000000000000127641474454370000271050ustar00rootroot00000000000000###################################################################### # # File: test/unit/internal/transfer/downloader/test_parallel.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import hashlib from concurrent.futures import ThreadPoolExecutor from io import BytesIO from unittest.mock import Mock import pytest from requests import RequestException def mock_download_response_factory(apiver_module, bucket, file_size: int = 0): hasher = hashlib.sha1() dummy_data = b'dummy' file_content = (dummy_data * (file_size // len(dummy_data) + 1))[:file_size] file_version = bucket.upload_bytes(file_content, f'dummy_file_{file_size}.txt') hasher.update(file_content) url = bucket.api.session.get_download_url_by_name(bucket.name, file_version.file_name) response = bucket.api.services.session.download_file_from_url(url).__enter__() return response, apiver_module.DownloadVersionFactory(bucket.api).from_response_headers( response.headers ) @pytest.fixture def thread_pool(): with ThreadPoolExecutor(max_workers=10) as executor: yield executor @pytest.fixture def output_file(): return BytesIO() @pytest.fixture def downloader(apiver_module, thread_pool): return apiver_module.ParallelDownloader( min_part_size=10, force_chunk_size=5, thread_pool=thread_pool, ) def test_download_empty_file(apiver_module, b2api, bucket, downloader, output_file): file_size = 0 mock_response, download_version = mock_download_response_factory( apiver_module, bucket, file_size=file_size ) mock_response.close = Mock(side_effect=mock_response.close) bytes_written, hash_hex = downloader.download( output_file, mock_response, download_version, b2api.session ) assert bytes_written == file_size assert hash_hex == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' assert output_file.getvalue() == b'' mock_response.close.assert_called_once() def test_download_file(apiver_module, b2api, bucket, downloader, output_file): file_size = 100 mock_response, download_version = mock_download_response_factory( apiver_module, bucket, file_size=file_size ) mock_response.close = Mock(side_effect=mock_response.close) bytes_written, hash_hex = downloader.download( output_file, mock_response, download_version, b2api.session ) assert bytes_written == file_size assert hash_hex == '7804df8c623573ccfc1993e04981006e5bc30383' assert output_file.getvalue() == b'dummy' * 20 mock_response.close.assert_called_once() def test_download_file__data_stream_error__in_first_response( apiver_module, b2api, bucket, downloader, output_file ): """ Test that the downloader handles a stream error in the first response. """ file_size = 100 mock_response, download_version = mock_download_response_factory( apiver_module, bucket, file_size=file_size ) def iter_content(chunk_size=1, decode_unicode=False): yield b'DUMMY' raise RequestException('stream error') yield # noqa mock_response.iter_content = iter_content bytes_written, hash_hex = downloader.download( output_file, mock_response, download_version, b2api.session ) assert bytes_written == file_size assert output_file.getvalue() == b'DUMMY' + b'dummy' * 19 def test_download_file__data_stream_error__persistent_errors( apiver_module, b2api, bucket, downloader, output_file ): file_size = 1000 mock_response, download_version = mock_download_response_factory( apiver_module, bucket, file_size=file_size ) # Ensure that follow-up requests also return errors def iter_content(chunk_size=1, decode_unicode=False): yield b'd' raise RequestException('stream error') mock_response.iter_content = iter_content bucket.api.services.session.download_file_from_url = Mock(return_value=mock_response) with pytest.raises(RequestException): downloader.download(output_file, mock_response, download_version, b2api.session) def test_download_file__data_stream_error__multiple_errors_recovery( apiver_module, b2api, bucket, downloader, output_file ): """Test downloader handles multiple half-failed requests and still downlaods entire file.""" # This works since each part is attempted up to 15 times before giving up file_size = 100 mock_response, download_version = mock_download_response_factory( apiver_module, bucket, file_size=file_size ) def first_iter_content(chunk_size=1, decode_unicode=False): yield mock_response.raw.read(1) raise RequestException('stream error') mock_response.iter_content = first_iter_content download_func = bucket.api.services.session.download_file_from_url def download_func_mock(*args, **kwargs): response = download_func(*args, **kwargs).__enter__() def iter_content(chunk_size=1, decode_unicode=False): yield response.raw.read(1).upper() raise RequestException('stream error') response.iter_content = iter_content return response bucket.api.services.session.download_file_from_url = download_func_mock bytes_written, hash_hex = downloader.download( output_file, mock_response, download_version, b2api.session ) assert bytes_written == file_size assert output_file.getvalue() == b'dUMMY' + b'DUMMY' * 19 b2-sdk-python-2.8.0/test/unit/replication/000077500000000000000000000000001474454370000204015ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/replication/conftest.py000066400000000000000000000027271474454370000226100ustar00rootroot00000000000000###################################################################### # # File: test/unit/replication/conftest.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from apiver_deps import ( Bucket, ReplicationConfiguration, ReplicationMonitor, ReplicationRule, ) @pytest.fixture def destination_bucket(b2api) -> Bucket: return b2api.create_bucket('destination-bucket', 'allPublic') @pytest.fixture def source_bucket(b2api, destination_bucket) -> Bucket: bucket = b2api.create_bucket('source-bucket', 'allPublic') bucket.replication = ReplicationConfiguration( rules=[ ReplicationRule( destination_bucket_id=destination_bucket.id_, name='name', file_name_prefix='folder/', ), ], source_key_id='hoho|trololo', ) return bucket @pytest.fixture def test_file(tmpdir) -> str: file = tmpdir.join('test.txt') file.write('whatever') return file @pytest.fixture def test_file_reversed(tmpdir) -> str: file = tmpdir.join('test-reversed.txt') file.write('revetahw') return file @pytest.fixture def monitor(source_bucket) -> ReplicationMonitor: return ReplicationMonitor( source_bucket, rule=source_bucket.replication.rules[0], ) b2-sdk-python-2.8.0/test/unit/replication/test_monitoring.py000066400000000000000000000204401474454370000241770ustar00rootroot00000000000000###################################################################### # # File: test/unit/replication/test_monitoring.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from apiver_deps import ( SSE_B2_AES, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, FileRetentionSetting, ReplicationScanResult, RetentionMode, ) SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_key', key_id='some-id'), ) RETENTION_GOVERNANCE = FileRetentionSetting(RetentionMode.GOVERNANCE, retain_until=1) DEFAULT_REPLICATION_RESULT = dict( source_replication_status=None, source_has_hide_marker=False, source_encryption_mode=EncryptionMode.NONE, source_has_large_metadata=False, source_has_file_retention=False, source_has_legal_hold=False, destination_replication_status=None, metadata_differs=None, hash_differs=None, ) def test_iter_pairs(source_bucket, destination_bucket, test_file, monitor): source_file = source_bucket.upload_local_file(test_file, 'folder/test.txt') source_subfolder_file = source_bucket.upload_local_file(test_file, 'folder/subfolder/test.txt') destination_subfolder_file = destination_bucket.upload_local_file( test_file, 'folder/subfolder/test.txt' ) destination_other_file = destination_bucket.upload_local_file( test_file, 'folder/subfolder/test2.txt' ) pairs = [ ( source_path and 'folder/' + source_path.relative_path, destination_path and 'folder/' + destination_path.relative_path, ) for source_path, destination_path in monitor.iter_pairs() ] assert set(pairs) == { (source_file.file_name, None), (source_subfolder_file.file_name, destination_subfolder_file.file_name), (None, destination_other_file.file_name), } def test_scan_source(source_bucket, test_file, monitor): # upload various types of files to source and get a report files = [ source_bucket.upload_local_file(test_file, 'folder/test-1-1.txt'), source_bucket.upload_local_file(test_file, 'folder/test-1-2.txt'), source_bucket.upload_local_file(test_file, 'folder/test-2.txt', encryption=SSE_B2_AES), source_bucket.upload_local_file( test_file, 'not-in-folder.txt' ), # monitor should ignore this source_bucket.upload_local_file(test_file, 'folder/test-3.txt', encryption=SSE_C_AES), source_bucket.upload_local_file(test_file, 'folder/test-4.txt', encryption=SSE_C_AES), source_bucket.upload_local_file( test_file, 'folder/subfolder/test-5.txt', encryption=SSE_C_AES, file_retention=RETENTION_GOVERNANCE, ), source_bucket.upload_local_file( test_file, 'folder/test-large-meta.txt', file_info={ 'dummy-key': 'a' * 7000, }, ), source_bucket.upload_local_file( test_file, 'folder/test-large-meta-encrypted.txt', file_info={ 'dummy-key': 'a' * 2048, }, encryption=SSE_C_AES, ), ] report = monitor.scan(scan_destination=False) assert report.counter_by_status[ReplicationScanResult(**DEFAULT_REPLICATION_RESULT)] == 2 assert ( report.counter_by_status[ ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_encryption_mode': EncryptionMode.SSE_B2, } ) ] == 1 ) assert ( report.counter_by_status[ ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_encryption_mode': EncryptionMode.SSE_C, } ) ] == 2 ) assert ( report.counter_by_status[ ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_encryption_mode': EncryptionMode.SSE_C, 'source_has_file_retention': True, } ) ] == 1 ) assert ( report.counter_by_status[ ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_has_large_metadata': True, } ) ] == 1 ) assert ( report.counter_by_status[ ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_encryption_mode': EncryptionMode.SSE_C, 'source_has_large_metadata': True, } ) ] == 1 ) # ---- first and last ---- assert ( report.samples_by_status_first[ ReplicationScanResult( **DEFAULT_REPLICATION_RESULT, ) ][0] == files[0] ) assert ( report.samples_by_status_last[ ReplicationScanResult( **DEFAULT_REPLICATION_RESULT, ) ][0] == files[1] ) def test_scan_source_and_destination( source_bucket, destination_bucket, test_file, test_file_reversed, monitor ): _ = [ # match source_bucket.upload_local_file(test_file, 'folder/test-1.txt'), destination_bucket.upload_local_file(test_file, 'folder/test-1.txt'), # missing on destination source_bucket.upload_local_file(test_file, 'folder/test-2.txt'), # missing on source destination_bucket.upload_local_file(test_file, 'folder/test-3.txt'), # metadata differs source_bucket.upload_local_file( test_file, 'folder/test-4.txt', file_info={ 'haha': 'hoho', }, ), destination_bucket.upload_local_file( test_file, 'folder/test-4.txt', file_info={ 'hehe': 'hihi', }, ), # hash differs source_bucket.upload_local_file(test_file, 'folder/test-5.txt'), destination_bucket.upload_local_file(test_file_reversed, 'folder/test-5.txt'), ] report = monitor.scan(scan_destination=True) # match assert ( report.counter_by_status[ ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'metadata_differs': False, 'hash_differs': False, } ) ] == 1 ) # missing on destination assert ( report.counter_by_status[ ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'destination_replication_status': None, } ) ] == 1 ) # missing on source assert ( report.counter_by_status[ ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'source_replication_status': None, 'source_has_hide_marker': None, 'source_encryption_mode': None, 'source_has_large_metadata': None, 'source_has_file_retention': None, 'source_has_legal_hold': None, } ) ] == 1 ) # metadata differs assert ( report.counter_by_status[ ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'metadata_differs': True, 'hash_differs': False, } ) ] == 1 ) # hash differs assert ( report.counter_by_status[ ReplicationScanResult( **{ **DEFAULT_REPLICATION_RESULT, 'metadata_differs': False, 'hash_differs': True, } ) ] == 1 ) b2-sdk-python-2.8.0/test/unit/scan/000077500000000000000000000000001474454370000170145ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/scan/__init__.py000066400000000000000000000005101474454370000211210ustar00rootroot00000000000000###################################################################### # # File: test/unit/scan/__init__.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/scan/test_folder_traversal.py000066400000000000000000000753561474454370000240030ustar00rootroot00000000000000###################################################################### # # File: test/unit/scan/test_folder_traversal.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import codecs import os import pathlib import platform import re import shlex import sys from unittest.mock import MagicMock, patch import pytest from b2sdk._internal.scan.folder import LocalFolder from b2sdk._internal.scan.policies import ScanPoliciesManager from b2sdk._internal.scan.report import ProgressReport from b2sdk._internal.utils import fix_windows_path_limit class TestFolderTraversal: def test_flat_folder(self, tmp_path): # Create a directory structure below with initial scanning point at tmp_path/dir: # tmp_path # └── dir # ├── file1.txt # ├── file2.txt # └── file3.txt (tmp_path / 'dir').mkdir(parents=True) (tmp_path / 'dir' / 'file1.txt').write_text('content1') (tmp_path / 'dir' / 'file2.txt').write_text('content2') (tmp_path / 'dir' / 'file3.txt').write_text('content3') folder = LocalFolder(str(tmp_path / 'dir')) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'dir' / 'file1.txt')), fix_windows_path_limit(str(tmp_path / 'dir' / 'file2.txt')), fix_windows_path_limit(str(tmp_path / 'dir' / 'file3.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows', reason="Windows doesn't allow / or \\ in filenames", ) def test_invalid_name(self, tmp_path): # Create a directory structure below with initial scanning point at tmp_path/dir: # tmp_path # └── dir # ├── file1.txt # ├── subdir # │ └── file2.txt # ├── file\bad.txt # └── file[DEL]bad.txt (tmp_path / 'dir' / 'subdir').mkdir(parents=True) (tmp_path / 'dir' / 'file1.txt').write_text('content1') (tmp_path / 'dir' / 'subdir' / 'file2.txt').write_text('content2') (tmp_path / 'dir' / 'file\\bad.txt').write_text('bad1') (tmp_path / 'dir' / 'file\x7fbad.txt').write_text('bad2') reporter = ProgressReport(sys.stdout, False) folder = LocalFolder(str(tmp_path / 'dir')) local_paths = folder.all_files(reporter=reporter) absolute_paths = [path.absolute_path for path in list(local_paths)] assert reporter.has_errors_or_warnings() assert isinstance(reporter.warnings, list) assert sorted(reporter.warnings) == [ f"WARNING: '{tmp_path}/dir/file\\bad.txt' path contains invalid name (file names must not contain '\\'). Skipping.", f"WARNING: '{tmp_path}/dir/file\\x7fbad.txt' path contains invalid name (file names must not contain DEL). Skipping.", ] reporter.close() assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'dir' / 'file1.txt')), fix_windows_path_limit(str(tmp_path / 'dir' / 'subdir' / 'file2.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows' and (platform.python_implementation() == 'PyPy' or sys.version_info >= (3, 13)), reason=( 'PyPy on Windows force-decodes non-UTF-8 filenames, which makes it impossible to test this case. ' 'Python 3.13 does so similarly on Windows.' ), ) def test_invalid_unicode_filename(self, tmp_path): # Create a directory structure below with initial scanning point at tmp_path/dir: # tmp_path # └── dir # ├── file1.txt # └── XXX (invalid utf-8 filename) (tmp_path / 'dir').mkdir(parents=True) (tmp_path / 'dir' / 'file1.txt').write_text('content1') foreign_encoding = 'euc_jp' # test sanity check assert ( codecs.lookup(foreign_encoding).name != codecs.lookup(sys.getfilesystemencoding()).name ) invalid_utf8_path = os.path.join(bytes(tmp_path), b'dir', 'てすと'.encode(foreign_encoding)) try: with open(invalid_utf8_path, 'wb') as f: f.write(b'content2') except (OSError, UnicodeDecodeError): pytest.skip('Cannot create invalid UTF-8 filename on this platform') reporter = ProgressReport(sys.stdout, False) folder = LocalFolder(str(tmp_path / 'dir')) local_paths = folder.all_files(reporter=reporter) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'dir' / 'file1.txt')), ] assert reporter.has_errors_or_warnings() assert re.match( r"WARNING: '.+/dir/.+' path contains invalid name " r'\(file name must be valid Unicode, check locale\)\. Skipping\.', reporter.warnings[0], ) assert len(reporter.warnings) == 1 reporter.close() @pytest.mark.skipif( platform.system() == 'Windows', reason="Windows doesn't allow / or \\ in filenames", ) def test_invalid_directory_name(self, tmp_path): # Create a directory structure below with initial scanning point at tmp_path/dir: # tmp_path # └── dir # ├── file1.txt # └── dir\bad # └── file2.txt (tmp_path / 'dir').mkdir(parents=True) (tmp_path / 'dir' / 'file1.txt').write_text('content1') (tmp_path / 'dir' / 'dir\\bad').mkdir(parents=True) (tmp_path / 'dir' / 'dir\\bad' / 'file2.txt').write_text('content2') reporter = ProgressReport(sys.stdout, False) folder = LocalFolder(str(tmp_path / 'dir')) local_paths = folder.all_files(reporter=reporter) absolute_paths = [path.absolute_path for path in list(local_paths)] assert reporter.has_errors_or_warnings() assert reporter.warnings == [ f"WARNING: '{tmp_path}/dir/dir\\bad' path contains invalid name (file names must not contain '\\'). Skipping." ] reporter.close() assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'dir' / 'file1.txt')), ] def test_folder_with_subfolders(self, tmp_path): # Create a directory structure below with initial scanning point at tmp_path: # tmp_path # ├── dir1 # │ ├── file1.txt # │ └── file2.txt # └── dir2 # ├── file3.txt # └── file4.txt d1 = tmp_path / 'dir1' d1.mkdir() (d1 / 'file1.txt').write_text('content1') (d1 / 'file2.txt').write_text('content2') d2 = tmp_path / 'dir2' d2.mkdir() (d2 / 'file3.txt').write_text('content3') (d2 / 'file4.txt').write_text('content4') folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(d1 / 'file1.txt')), fix_windows_path_limit(str(d1 / 'file2.txt')), fix_windows_path_limit(str(d2 / 'file3.txt')), fix_windows_path_limit(str(d2 / 'file4.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) def test_folder_with_symlink_to_file(self, tmp_path): # Create a directory structure below with initial scanning point at tmp_path: # tmp_path # ├── dir # │ └── file.txt # └── symlink_file.txt -> dir/file.txt (tmp_path / 'dir').mkdir() file = tmp_path / 'dir' / 'file.txt' file.write_text('content') symlink_file = tmp_path / 'symlink_file.txt' symlink_file.symlink_to(file) folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(file)), fix_windows_path_limit(str(symlink_file)), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) @pytest.mark.timeout(5) def test_folder_with_circular_symlink(self, tmp_path): # Create a directory structure below with initial scanning point at tmp_path: # tmp_path # ├── dir # │ └── file.txt # └── symlink_dir -> dir (tmp_path / 'dir').mkdir() (tmp_path / 'dir' / 'file1.txt').write_text('content1') symlink_dir = tmp_path / 'dir' / 'symlink_dir' symlink_dir.symlink_to(tmp_path / 'dir', target_is_directory=True) folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'dir' / 'file1.txt')), fix_windows_path_limit(str(tmp_path / 'dir' / 'symlink_dir' / 'file1.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) @pytest.mark.timeout(5) def test_folder_with_symlink_to_parent(self, tmp_path): # Create a directory structure below with the scanning point at tmp_path/parent/child/: # tmp_path # ├── parent # │ ├── child # │ │ ├── file4.txt # │ │ └── grandchild # │ │ ├── file5.txt # │ │ └── symlink_dir -> ../../.. (symlink to tmp_path/parent) # │ └── file3.txt # ├── file1.txt # └── file2.txt (tmp_path / 'parent' / 'child' / 'grandchild').mkdir(parents=True) (tmp_path / 'file1.txt').write_text('content1') (tmp_path / 'file2.txt').write_text('content2') (tmp_path / 'parent' / 'file3.txt').write_text('content3') (tmp_path / 'parent' / 'child' / 'file4.txt').write_text('content4') (tmp_path / 'parent' / 'child' / 'grandchild' / 'file5.txt').write_text('content5') symlink_dir = tmp_path / 'parent' / 'child' / 'grandchild' / 'symlink_dir' symlink_dir.symlink_to(tmp_path / 'parent', target_is_directory=True) folder = LocalFolder(str(tmp_path / 'parent' / 'child')) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'parent' / 'child' / 'file4.txt')), fix_windows_path_limit(str(tmp_path / 'parent' / 'child' / 'grandchild' / 'file5.txt')), fix_windows_path_limit( str( tmp_path / 'parent' / 'child' / 'grandchild' / 'symlink_dir' / 'child' / 'file4.txt' ) ), fix_windows_path_limit( str( tmp_path / 'parent' / 'child' / 'grandchild' / 'symlink_dir' / 'child' / 'grandchild' / 'file5.txt' ) ), fix_windows_path_limit( str(tmp_path / 'parent' / 'child' / 'grandchild' / 'symlink_dir' / 'file3.txt') ), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) @pytest.mark.timeout(5) def test_root_short_loop(self, tmp_path): # Create a symlink to the tmp_path directory itself # tmp_path # └── tmp_path_symlink -> tmp_path tmp_path_symlink = tmp_path / 'tmp_path_symlink' tmp_path_symlink.symlink_to(tmp_path, target_is_directory=True) folder = LocalFolder(str(tmp_path_symlink)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) @pytest.mark.timeout(5) def test_root_parent_loop(self, tmp_path): # Create a symlink that points to the parent of the initial scanning point # tmp_path # └── start # ├── file.txt # └── symlink -> tmp_path (tmp_path / 'start').mkdir() (tmp_path / 'start' / 'file.txt').write_text('content') (tmp_path / 'start' / 'symlink').symlink_to(tmp_path, target_is_directory=True) folder = LocalFolder(str(tmp_path / 'start')) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'start' / 'file.txt')), fix_windows_path_limit(str(tmp_path / 'start' / 'symlink' / 'start' / 'file.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) def test_symlink_that_points_deeper(self, tmp_path): # Create a directory structure with a symlink that points to a deeper directory # tmp_path # ├── a # │ └── a.txt # └── b # ├── c # │ └── c.txt # └── d # ├── d.txt # └── e # └── e.txt # ├── f # │ └── f.txt # └── symlink -> b/d/e (tmp_path / 'a').mkdir() (tmp_path / 'a' / 'a.txt').write_text('a') (tmp_path / 'b' / 'c').mkdir(parents=True) (tmp_path / 'b' / 'c' / 'c.txt').write_text('c') (tmp_path / 'b' / 'd' / 'e').mkdir(parents=True) (tmp_path / 'b' / 'd' / 'd.txt').write_text('d') (tmp_path / 'b' / 'd' / 'e' / 'e.txt').write_text('e') (tmp_path / 'f').mkdir() (tmp_path / 'f' / 'f.txt').write_text('f') (tmp_path / 'symlink').symlink_to(tmp_path / 'b' / 'd' / 'e', target_is_directory=True) folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'a' / 'a.txt')), fix_windows_path_limit(str(tmp_path / 'b' / 'c' / 'c.txt')), fix_windows_path_limit(str(tmp_path / 'b' / 'd' / 'd.txt')), fix_windows_path_limit(str(tmp_path / 'b' / 'd' / 'e' / 'e.txt')), fix_windows_path_limit(str(tmp_path / 'f' / 'f.txt')), fix_windows_path_limit(str(tmp_path / 'symlink' / 'e.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) def test_symlink_that_points_up(self, tmp_path): # Create a directory structure with a symlink that points to a upper directory # tmp_path # ├── a # │ └── a.txt # └── b # ├── c # │ └── c.txt # └── d # ├── d.txt # └── e # ├── symlink -> ../../a # └── e.txt (tmp_path / 'a').mkdir() (tmp_path / 'a' / 'a.txt').write_text('a') (tmp_path / 'b' / 'c').mkdir(parents=True) (tmp_path / 'b' / 'c' / 'c.txt').write_text('c') (tmp_path / 'b' / 'd' / 'e').mkdir(parents=True) (tmp_path / 'b' / 'd' / 'd.txt').write_text('d') (tmp_path / 'b' / 'd' / 'e' / 'e.txt').write_text('e') (tmp_path / 'b' / 'd' / 'e' / 'symlink').symlink_to( tmp_path / 'a', target_is_directory=True ) folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'a' / 'a.txt')), fix_windows_path_limit(str(tmp_path / 'b' / 'c' / 'c.txt')), fix_windows_path_limit(str(tmp_path / 'b' / 'd' / 'd.txt')), fix_windows_path_limit(str(tmp_path / 'b' / 'd' / 'e' / 'e.txt')), fix_windows_path_limit(str(tmp_path / 'b' / 'd' / 'e' / 'symlink' / 'a.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) @pytest.mark.timeout(5) def test_elaborate_infinite_loop(self, tmp_path): # Create a directory structure with an elaborate infinite loop of symlinks # tmp_path # ├── a # │ └── a.txt # ├── b -> c # ├── c -> d # ├── d -> e # ├── e -> b # └── f # └── f.txt (tmp_path / 'a').mkdir() (tmp_path / 'a' / 'a.txt').write_text('a') (tmp_path / 'b').symlink_to('c') (tmp_path / 'c').symlink_to('d') (tmp_path / 'd').symlink_to('e') (tmp_path / 'e').symlink_to('b') (tmp_path / 'f').mkdir() (tmp_path / 'f' / 'f.txt').write_text('f') folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'a' / 'a.txt')), fix_windows_path_limit(str(tmp_path / 'f' / 'f.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) def test_valid_symlink_pattern_where_the_link_goes_down_and_up(self, tmp_path): # tmp_path # ├── a # │ └── a.txt # ├── b -> c/d # ├── c # │ └── d # │ └── b.txt # ├── d -> e # ├── e # │ └── e.txt # └── f # └── f.txt (tmp_path / 'a').mkdir() (tmp_path / 'a' / 'a.txt').write_text('a') (tmp_path / 'b').symlink_to(tmp_path / 'c' / 'd', target_is_directory=True) (tmp_path / 'c').mkdir() (tmp_path / 'c' / 'd').mkdir() (tmp_path / 'c' / 'd' / 'b.txt').write_text('b') (tmp_path / 'd').symlink_to(tmp_path / 'e', target_is_directory=True) (tmp_path / 'e').mkdir() (tmp_path / 'e' / 'e.txt').write_text('e') (tmp_path / 'f').mkdir() (tmp_path / 'f' / 'f.txt').write_text('f') folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'a' / 'a.txt')), fix_windows_path_limit(str(tmp_path / 'b' / 'b.txt')), fix_windows_path_limit(str(tmp_path / 'c' / 'd' / 'b.txt')), fix_windows_path_limit(str(tmp_path / 'd' / 'e.txt')), fix_windows_path_limit(str(tmp_path / 'e' / 'e.txt')), fix_windows_path_limit(str(tmp_path / 'f' / 'f.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) def test_valid_symlink_pattern_where_the_link_goes_up_and_down(self, tmp_path): # Create a directory structure with a valid symlink pattern where the link goes up and down # tmp_path # ├── a # │ └── a.txt # ├── b # │ └── c -> ../d # ├── d # │ └── e # │ └── f # │ └── f.txt # └── t.txt (tmp_path / 'a').mkdir() (tmp_path / 'a' / 'a.txt').write_text('a') (tmp_path / 'b').mkdir() (tmp_path / 'b' / 'c').symlink_to(tmp_path / 'd', target_is_directory=True) (tmp_path / 'd').mkdir() (tmp_path / 'd' / 'e').mkdir() (tmp_path / 'd' / 'e' / 'f').mkdir() (tmp_path / 'd' / 'e' / 'f' / 'f.txt').write_text('f') (tmp_path / 't.txt').write_text('t') folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'a' / 'a.txt')), fix_windows_path_limit(str(tmp_path / 'b' / 'c' / 'e' / 'f' / 'f.txt')), fix_windows_path_limit(str(tmp_path / 'd' / 'e' / 'f' / 'f.txt')), fix_windows_path_limit(str(tmp_path / 't.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) @pytest.mark.timeout(5) def test_loop_that_goes_down_and_up(self, tmp_path): # Create a directory structure with a loop that goes down and up # tmp_path # ├── a # │ └── a.txt # ├── b -> c/d # ├── c # │ └── d -> ../e # ├── e -> b # └── f # └── f.txt (tmp_path / 'a').mkdir() (tmp_path / 'a' / 'a.txt').write_text('a') (tmp_path / 'b').symlink_to(tmp_path / 'c' / 'd', target_is_directory=True) (tmp_path / 'c').mkdir() (tmp_path / 'c' / 'd').symlink_to(tmp_path / 'e', target_is_directory=True) (tmp_path / 'e').symlink_to('b') (tmp_path / 'f').mkdir() (tmp_path / 'f' / 'f.txt').write_text('f') folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'a' / 'a.txt')), fix_windows_path_limit(str(tmp_path / 'f' / 'f.txt')), ] @pytest.mark.skipif( platform.system() == 'Windows' and platform.python_implementation() == 'PyPy', reason='Symlinks not supported on PyPy/Windows', ) @pytest.mark.timeout(5) def test_loop_that_goes_up_and_down(self, tmp_path): # Create a directory structure with a loop that goes up and down # tmp_path # ├── a # │ └── a.txt # ├── b # │ └── c -> ../d # ├── d # │ └── e # │ └── f -> ../../b/c # └── g # └── g.txt (tmp_path / 'a').mkdir() (tmp_path / 'a' / 'a.txt').write_text('a') (tmp_path / 'b').mkdir() (tmp_path / 'b' / 'c').symlink_to(tmp_path / 'd', target_is_directory=True) (tmp_path / 'd').mkdir() (tmp_path / 'd' / 'e').mkdir() (tmp_path / 'd' / 'e' / 'f').symlink_to(tmp_path / 'b' / 'c', target_is_directory=True) (tmp_path / 'g').mkdir() (tmp_path / 'g' / 'g.txt').write_text('g') folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=MagicMock()) absolute_paths = [path.absolute_path for path in list(local_paths)] assert absolute_paths == [ fix_windows_path_limit(str(tmp_path / 'a' / 'a.txt')), fix_windows_path_limit(str(tmp_path / 'g' / 'g.txt')), ] def test_folder_all_files__dir_excluded_by_regex(self, tmp_path): """ bar$ regex should exclude bar directory and all files inside it """ d1_dir = tmp_path / 'd1' d1_dir.mkdir() (d1_dir / 'file1.txt').touch() bar_dir = tmp_path / 'bar' bar_dir.mkdir() (bar_dir / 'file2.txt').touch() scan_policy = ScanPoliciesManager(exclude_dir_regexes=['bar$']) folder = LocalFolder(tmp_path) local_paths = folder.all_files(reporter=None, policies_manager=scan_policy) absolute_paths = [path.absolute_path for path in local_paths] assert absolute_paths == [ fix_windows_path_limit(str(d1_dir / 'file1.txt')), ] def test_excluded_no_access_check(self, tmp_path): """Test that a directory/file is not checked for access if it is excluded.""" # Create directories and files excluded_dir = tmp_path / 'excluded_dir' excluded_dir.mkdir() excluded_file = excluded_dir / 'excluded_file.txt' excluded_file.touch() included_dir = tmp_path / 'included_dir' included_dir.mkdir() (included_dir / 'excluded_file.txt').touch() # Setup exclusion regex that matches the excluded directory/file name scan_policy = ScanPoliciesManager( exclude_dir_regexes=[r'excluded_dir$'], exclude_file_regexes=[r'.*excluded_file.txt'] ) reporter = ProgressReport(sys.stdout, False) # Patch os.access to monitor if it is called on the excluded file with patch('os.access', MagicMock(return_value=True)) as mocked_access: folder = LocalFolder(str(tmp_path)) list(folder.all_files(reporter=reporter, policies_manager=scan_policy)) # Verify os.access was not called for the excluded directory/file mocked_access.assert_not_called() reporter.close() @pytest.mark.skipif( platform.system() == 'Windows', reason='Unix-only filesystem permissions are tested', ) def test_dir_without_exec_permission(self, tmp_path, fs_perm_tool): """Test that a excluded directory/file without permissions emits warnings.""" no_perm_dir = tmp_path / 'no_perm_dir' no_perm_dir.mkdir() (no_perm_dir / 'file.txt').touch() (no_perm_dir / 'file2.txt').touch() # chmod -x no_perm_dir no_perm_dir.chmod(0o600) scan_policy = ScanPoliciesManager() reporter = ProgressReport(sys.stdout, False) folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=reporter, policies_manager=scan_policy) absolute_paths = [path.absolute_path for path in local_paths] assert not absolute_paths # Check that no access warnings are issued for the excluded directory/file assert set(reporter.warnings) == { f'WARNING: {tmp_path/"no_perm_dir/file.txt"} could not be accessed (no permissions to read?)', f'WARNING: {tmp_path/"no_perm_dir/file2.txt"} could not be accessed (no permissions to read?)', } reporter.close() def test_without_permissions(self, tmp_path, fs_perm_tool): """Test that a excluded directory/file without permissions emits warnings.""" no_perm_dir = tmp_path / 'no_perm_dir' no_perm_dir.mkdir() (no_perm_dir / 'file.txt').touch() included_dir = tmp_path / 'included_dir' included_dir.mkdir() (included_dir / 'no_perm_file.txt').touch() (included_dir / 'included_file.txt').touch() # Modify directory permissions to simulate lack of access fs_perm_tool.deny_access(included_dir / 'no_perm_file.txt') fs_perm_tool.deny_access(no_perm_dir) scan_policy = ScanPoliciesManager() reporter = ProgressReport(sys.stdout, False) folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=reporter, policies_manager=scan_policy) absolute_paths = [pathlib.Path(path.absolute_path) for path in local_paths] # Check that only included_dir/included_file.txt was return assert {path.name for path in absolute_paths} == {'included_file.txt'} def s(p): # shlex.quote works differently depending if its on windows or unix return shlex.quote(str(p)) # Check that no access warnings are issued for the excluded directory/file assert set(reporter.warnings) == { f'WARNING: {s(tmp_path / "no_perm_dir")} could not be accessed (no permissions to read?)', f'WARNING: {s(tmp_path / "included_dir/no_perm_file.txt")} could not be accessed (no permissions to read?)', } reporter.close() def test_excluded_without_permissions(self, tmp_path, fs_perm_tool): """Test that a excluded directory/file without permissions is not processed and no warning is issued.""" no_perm_dir = tmp_path / 'no_perm_dir' no_perm_dir.mkdir() (no_perm_dir / 'file.txt').touch() included_dir = tmp_path / 'included_dir' included_dir.mkdir() (included_dir / 'no_perm_file.txt').touch() (included_dir / 'included_file.txt').touch() # Modify directory permissions to simulate lack of access fs_perm_tool.deny_access(included_dir / 'no_perm_file.txt') fs_perm_tool.deny_access(no_perm_dir) scan_policy = ScanPoliciesManager( exclude_dir_regexes=[r'no_perm_dir$'], exclude_file_regexes=[r'.*no_perm_file.txt'] ) reporter = ProgressReport(sys.stdout, False) folder = LocalFolder(str(tmp_path)) local_paths = folder.all_files(reporter=reporter, policies_manager=scan_policy) absolute_paths = [path.absolute_path for path in local_paths] # Check that only included_dir/included_file.txt was return assert any('included_file.txt' in path for path in absolute_paths) # Check that no access warnings are issued for the excluded directory/file assert not reporter.warnings reporter.close() b2-sdk-python-2.8.0/test/unit/scan/test_scan_policies.py000066400000000000000000000057041474454370000232460ustar00rootroot00000000000000###################################################################### # # File: test/unit/scan/test_scan_policies.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import re import pytest from apiver_deps import ScanPoliciesManager from apiver_deps_exception import InvalidArgument class TestScanPoliciesManager: def test_include_file_regexes_without_exclude(self): kwargs = {'include_file_regexes': '.*'} # valid regex with pytest.raises(InvalidArgument): ScanPoliciesManager(**kwargs) @pytest.mark.parametrize( 'param,exception', [ pytest.param( 'exclude_dir_regexes', InvalidArgument, marks=pytest.mark.apiver(from_ver=2) ), pytest.param( 'exclude_file_regexes', InvalidArgument, marks=pytest.mark.apiver(from_ver=2) ), pytest.param( 'include_file_regexes', InvalidArgument, marks=pytest.mark.apiver(from_ver=2) ), pytest.param('exclude_dir_regexes', re.error, marks=pytest.mark.apiver(to_ver=1)), pytest.param('exclude_file_regexes', re.error, marks=pytest.mark.apiver(to_ver=1)), pytest.param('include_file_regexes', re.error, marks=pytest.mark.apiver(to_ver=1)), ], ) def test_illegal_regex(self, param, exception): kwargs = { 'exclude_dir_regexes': '.*', 'exclude_file_regexes': '.*', 'include_file_regexes': '.*', param: '*', # invalid regex } with pytest.raises(exception): ScanPoliciesManager(**kwargs) @pytest.mark.parametrize( 'param,exception', [ pytest.param( 'exclude_modified_before', InvalidArgument, marks=pytest.mark.apiver(from_ver=2) ), pytest.param( 'exclude_modified_after', InvalidArgument, marks=pytest.mark.apiver(from_ver=2) ), pytest.param('exclude_modified_before', ValueError, marks=pytest.mark.apiver(to_ver=1)), pytest.param('exclude_modified_after', ValueError, marks=pytest.mark.apiver(to_ver=1)), ], ) def test_illegal_timestamp(self, param, exception): kwargs = { 'exclude_modified_before': 1, 'exclude_modified_after': 2, param: -1.0, # invalid range param } with pytest.raises(exception): ScanPoliciesManager(**kwargs) @pytest.mark.apiver(from_ver=2) def test_re_pattern_argument_support(self): kwargs = { param: (re.compile(r'.*'),) for param in ( 'exclude_dir_regexes', 'exclude_file_regexes', 'include_file_regexes', ) } ScanPoliciesManager(**kwargs) b2-sdk-python-2.8.0/test/unit/stream/000077500000000000000000000000001474454370000173635ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/stream/__init__.py000066400000000000000000000004471474454370000215010ustar00rootroot00000000000000###################################################################### # # File: test/unit/stream/__init__.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### b2-sdk-python-2.8.0/test/unit/stream/test_progress.py000066400000000000000000000034061474454370000226430ustar00rootroot00000000000000###################################################################### # # File: test/unit/stream/test_progress.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import io from unittest.mock import Mock from apiver_deps import ReadingStreamWithProgress def test_reading_stream_with_progress(tmp_path): stream = io.BytesIO(b'1234567890') progress_listener = Mock() with ReadingStreamWithProgress(stream, progress_listener=progress_listener) as wrapped_stream: assert wrapped_stream.read(1) == b'1' assert wrapped_stream.read(2) == b'23' assert wrapped_stream.read(3) == b'456' assert progress_listener.bytes_completed.call_count == 3 assert wrapped_stream.bytes_completed == 6 assert not stream.closed def test_reading_stream_with_progress__not_closing_wrapped_stream(tmp_path): stream = io.BytesIO(b'1234567890') progress_listener = Mock() with ReadingStreamWithProgress(stream, progress_listener=progress_listener) as wrapped_stream: assert wrapped_stream.read() assert not stream.closed def test_reading_stream_with_progress__closed_proxy(tmp_path): """ Test that the wrapped stream is closed when the original stream is closed. This is important for Python 3.13+ to prevent: 'Exception ignored in: ' messages. """ stream = io.BytesIO(b'1234567890') progress_listener = Mock() wrapped_stream = ReadingStreamWithProgress(stream, progress_listener=progress_listener) assert not stream.closed stream.close() assert wrapped_stream.closed b2-sdk-python-2.8.0/test/unit/sync/000077500000000000000000000000001474454370000170445ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/sync/__init__.py000066400000000000000000000005101474454370000211510ustar00rootroot00000000000000###################################################################### # # File: test/unit/sync/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/sync/fixtures.py000066400000000000000000000041501474454370000212670ustar00rootroot00000000000000###################################################################### # # File: test/unit/sync/fixtures.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import apiver_deps import pytest from apiver_deps import ( DEFAULT_SCAN_MANAGER, POLICY_MANAGER, CompareVersionMode, KeepOrDeleteMode, NewerFileSyncMode, Synchronizer, UploadMode, ) @pytest.fixture(scope='session') def synchronizer_factory(): def get_synchronizer( policies_manager=DEFAULT_SCAN_MANAGER, dry_run=False, allow_empty_source=False, newer_file_mode=NewerFileSyncMode.RAISE_ERROR, keep_days_or_delete=KeepOrDeleteMode.NO_DELETE, keep_days=None, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=None, sync_policy_manager=POLICY_MANAGER, upload_mode=UploadMode.FULL, absolute_minimum_part_size=None, ): kwargs = {} if apiver_deps.V < 2: assert upload_mode == UploadMode.FULL, 'upload_mode not supported in apiver < 2' assert ( absolute_minimum_part_size is None ), 'absolute_minimum_part_size not supported in apiver < 2' else: kwargs = dict( upload_mode=upload_mode, absolute_minimum_part_size=absolute_minimum_part_size, ) return Synchronizer( 1, policies_manager=policies_manager, dry_run=dry_run, allow_empty_source=allow_empty_source, newer_file_mode=newer_file_mode, keep_days_or_delete=keep_days_or_delete, keep_days=keep_days, compare_version_mode=compare_version_mode, compare_threshold=compare_threshold, sync_policy_manager=sync_policy_manager, **kwargs, ) return get_synchronizer @pytest.fixture def synchronizer(synchronizer_factory): return synchronizer_factory() b2-sdk-python-2.8.0/test/unit/sync/test_exception.py000066400000000000000000000044671474454370000224660ustar00rootroot00000000000000###################################################################### # # File: test/unit/sync/test_exception.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from apiver_deps_exception import ( EnvironmentEncodingError, IncompleteSync, InvalidArgument, UnsupportedFilename, check_invalid_argument, ) class TestSyncExceptions: def test_environment_encoding_error(self): try: raise EnvironmentEncodingError('fred', 'george') except EnvironmentEncodingError as e: assert ( str(e) == """file name fred cannot be decoded with system encoding (george). We think this is an environment error which you should workaround by setting your system encoding properly, for example like this: export LANG=en_US.UTF-8""" ), str(e) def test_invalid_argument(self): try: raise InvalidArgument('param', 'message') except InvalidArgument as e: assert str(e) == 'param message', str(e) def test_incomplete_sync(self): try: raise IncompleteSync() except IncompleteSync as e: assert str(e) == 'Incomplete sync: ', str(e) def test_unsupportedfilename_error(self): try: raise UnsupportedFilename('message', 'filename') except UnsupportedFilename as e: assert str(e) == 'message: filename', str(e) class TestCheckInvalidArgument: def test_custom_message(self): with pytest.raises(InvalidArgument): try: with check_invalid_argument('param', 'an error occurred', RuntimeError): raise RuntimeError() except InvalidArgument as exc: assert str(exc) == 'param an error occurred' raise def test_message_from_exception(self): with pytest.raises(InvalidArgument): try: with check_invalid_argument('param', '', RuntimeError): raise RuntimeError('an error occurred') except InvalidArgument as exc: assert str(exc) == 'param an error occurred' raise b2-sdk-python-2.8.0/test/unit/sync/test_sync.py000066400000000000000000001147461474454370000214460ustar00rootroot00000000000000###################################################################### # # File: test/unit/sync/test_sync.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from collections import defaultdict from contextlib import ExitStack from enum import Enum from functools import partial from unittest import mock import pytest from apiver_deps import ( AbstractSyncEncryptionSettingsProvider, B2DownloadAction, CompareVersionMode, CopySource, FileVersion, IncrementalHexDigester, KeepOrDeleteMode, NewerFileSyncMode, SyncPolicyManager, UploadSourceLocalFile, UploadSourceLocalFileRange, UpPolicy, ) from apiver_deps_exception import DestFileNewer, InvalidArgument from ..fixtures.folder import * from .fixtures import * DAY = 86400000 # milliseconds TODAY = DAY * 100 # an arbitrary reference time for testing class IllegalEnum(Enum): ILLEGAL = 5100 class TestSynchronizer: @pytest.fixture(autouse=True) def setup(self, folder_factory, mocker, apiver): self.folder_factory = folder_factory self.local_folder_factory = partial(folder_factory, 'local') self.b2_folder_factory = partial(folder_factory, 'b2') self.reporter = mocker.MagicMock() self.apiver = apiver def _make_folder_sync_actions(self, synchronizer, *args, **kwargs): if self.apiver in ['v0', 'v1']: return synchronizer.make_folder_sync_actions(*args, **kwargs) return synchronizer._make_folder_sync_actions(*args, **kwargs) def assert_folder_sync_actions(self, synchronizer, src_folder, dst_folder, expected_actions): """ Checks the actions generated for one file. The file may or may not exist at the source, and may or may not exist at the destination. The source and destination files may have multiple versions. """ actions = list( self._make_folder_sync_actions( synchronizer, src_folder, dst_folder, TODAY, self.reporter, ) ) assert expected_actions == [str(a) for a in actions] @pytest.mark.apiver(to_ver=0) @pytest.mark.parametrize( 'args', [ {'newer_file_mode': IllegalEnum.ILLEGAL}, {'keep_days_or_delete': IllegalEnum.ILLEGAL}, ], ids=[ 'newer_file_mode', 'keep_days_or_delete', ], ) def test_illegal_args_up_to_v0(self, synchronizer_factory, apiver, args): from apiver_deps_exception import CommandError with pytest.raises(CommandError): synchronizer_factory(**args) @pytest.mark.apiver(from_ver=1) @pytest.mark.parametrize( 'args', [ {'newer_file_mode': IllegalEnum.ILLEGAL}, {'keep_days_or_delete': IllegalEnum.ILLEGAL}, ], ids=[ 'newer_file_mode', 'keep_days_or_delete', ], ) def test_illegal_args_up_v1_and_up(self, synchronizer_factory, apiver, args): with pytest.raises(InvalidArgument): synchronizer_factory(**args) def test_illegal(self, synchronizer): with pytest.raises(ValueError): src = self.local_folder_factory() dst = self.local_folder_factory() self.assert_folder_sync_actions(synchronizer, src, dst, []) # src: absent, dst: absent @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_empty(self, synchronizer, src_type, dst_type): src = self.folder_factory(src_type) dst = self.folder_factory(dst_type) self.assert_folder_sync_actions(synchronizer, src, dst, []) # # src: present, dst: absent @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 100)']), ('b2', 'local', ['b2_download(folder/a.txt, id_a_100, /dir/a.txt, 100)']), ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_100, folder/a.txt, 100)']), ], ) def test_not_there(self, synchronizer, src_type, dst_type, expected): src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,expected', [ ('local', ['b2_upload(/dir/directory/a.txt, folder/directory/a.txt, 100)']), ('b2', ['b2_copy(folder/directory/a.txt, id_d_100, folder/directory/a.txt, 100)']), ], ) def test_dir_not_there_b2_keepdays( self, synchronizer_factory, src_type, expected ): # reproduces issue 220 src = self.folder_factory(src_type, ('directory/a.txt', [100])) dst = self.b2_folder_factory() synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,expected', [ ('local', ['b2_upload(/dir/directory/a.txt, folder/directory/a.txt, 100)']), ('b2', ['b2_copy(folder/directory/a.txt, id_d_100, folder/directory/a.txt, 100)']), ], ) def test_dir_not_there_b2_delete( self, synchronizer_factory, src_type, expected ): # reproduces issue 220 src = self.folder_factory(src_type, ('directory/a.txt', [100])) dst = self.b2_folder_factory() synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) self.assert_folder_sync_actions(synchronizer, src, dst, expected) # # src: absent, dst: present @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_no_delete(self, synchronizer, src_type, dst_type): src = self.folder_factory(src_type) dst = self.folder_factory(dst_type, ('a.txt', [100])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), ('b2', 'local', ['local_delete(/dir/a.txt)']), ('b2', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), ], ) def test_delete(self, synchronizer_factory, src_type, dst_type, expected): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type) dst = self.folder_factory(dst_type, ('a.txt', [100])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), ('b2', 'local', ['local_delete(/dir/a.txt)']), ('b2', 'b2', ['b2_delete(folder/a.txt, id_a_100, )']), ], ) def test_delete_large(self, synchronizer_factory, src_type, dst_type, expected): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type) dst = self.folder_factory(dst_type, ('a.txt', [100], 10737418240)) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_delete_multiple_versions(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [100, 200])) expected = [ 'b2_delete(folder/a.txt, id_a_100, )', 'b2_delete(folder/a.txt, id_a_200, (old version))', ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_delete_hide_b2_multiple_versions(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) expected = [ 'b2_hide(folder/a.txt)', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))', ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_delete_hide_b2_multiple_versions_old(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY, TODAY - 5 * DAY])) expected = [ 'b2_hide(folder/a.txt)', 'b2_delete(folder/a.txt, id_a_8208000000, (old version))', ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep(self, synchronizer, src_type): src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep_days(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) expected = ['b2_delete(folder/a.txt, id_a_8294400000, (old version))'] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep_days_one_old( self, synchronizer_factory, src_type ): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=5 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory( ('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) ) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep_days_two_old( self, synchronizer_factory, src_type ): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory( ('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) ) expected = ['b2_delete(folder/a.txt, id_a_8121600000, (old version))'] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep_days_delete_hide_marker( self, synchronizer_factory, src_type ): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory( ('a.txt', [-(TODAY - 2 * DAY), TODAY - 4 * DAY, TODAY - 6 * DAY]) ) expected = [ 'b2_delete(folder/a.txt, id_a_8467200000, (hide marker))', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))', 'b2_delete(folder/a.txt, id_a_8121600000, (old version))', ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_keep_days_old_delete( self, synchronizer_factory, src_type ): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [-TODAY + 2 * DAY, TODAY - 4 * DAY])) expected = [ 'b2_delete(folder/a.txt, id_a_8467200000, (hide marker))', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))', ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_already_hidden_multiple_versions_delete(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type) dst = self.b2_folder_factory(('a.txt', [-TODAY, TODAY - 2 * DAY, TODAY - 4 * DAY])) expected = [ 'b2_delete(folder/a.txt, id_a_8640000000, (hide marker))', 'b2_delete(folder/a.txt, id_a_8467200000, (old version))', 'b2_delete(folder/a.txt, id_a_8294400000, (old version))', ] self.assert_folder_sync_actions(synchronizer, src, dst, expected) # # src same as dst @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_same(self, synchronizer, src_type, dst_type): src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type, ('a.txt', [100])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_same_leave_old_version(self, synchronizer, src_type): src = self.folder_factory(src_type, ('a.txt', [TODAY])) dst = self.b2_folder_factory(('a.txt', [TODAY, TODAY - 3 * DAY])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_same_clean_old_version(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type, ('a.txt', [TODAY - 3 * DAY])) dst = self.b2_folder_factory(('a.txt', [TODAY - 3 * DAY, TODAY - 4 * DAY])) expected = ['b2_delete(folder/a.txt, id_a_8294400000, (old version))'] self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_keep_days_no_change_with_old_file(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=1 ) src = self.folder_factory(src_type, ('a.txt', [TODAY - 3 * DAY])) dst = self.b2_folder_factory(('a.txt', [TODAY - 3 * DAY])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type', [ 'local', 'b2', ], ) def test_same_delete_old_versions(self, synchronizer_factory, src_type): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type, ('a.txt', [TODAY])) dst = self.b2_folder_factory(('a.txt', [TODAY, TODAY - 3 * DAY])) expected = ['b2_delete(folder/a.txt, id_a_8380800000, (old version))'] self.assert_folder_sync_actions(synchronizer, src, dst, expected) # # src newer than dst @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 200)']), ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)']), ], ) def test_newer(self, synchronizer, src_type, dst_type, expected): src = self.folder_factory(src_type, ('a.txt', [200])) dst = self.folder_factory(dst_type, ('a.txt', [100])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,expected', [ ( 'local', [ 'b2_upload(/dir/a.txt, folder/a.txt, 8640000000)', 'b2_delete(folder/a.txt, id_a_8208000000, (old version))', ], ), ( 'b2', [ 'b2_copy(folder/a.txt, id_a_8640000000, folder/a.txt, 8640000000)', 'b2_delete(folder/a.txt, id_a_8208000000, (old version))', ], ), ], ) def test_newer_clean_old_versions(self, synchronizer_factory, src_type, expected): synchronizer = synchronizer_factory( keep_days_or_delete=KeepOrDeleteMode.KEEP_BEFORE_DELETE, keep_days=2 ) src = self.folder_factory(src_type, ('a.txt', [TODAY])) dst = self.b2_folder_factory(('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY, TODAY - 5 * DAY])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,expected', [ ( 'local', [ 'b2_upload(/dir/a.txt, folder/a.txt, 8640000000)', 'b2_delete(folder/a.txt, id_a_8553600000, (old version))', 'b2_delete(folder/a.txt, id_a_8380800000, (old version))', ], ), ( 'b2', [ 'b2_copy(folder/a.txt, id_a_8640000000, folder/a.txt, 8640000000)', 'b2_delete(folder/a.txt, id_a_8553600000, (old version))', 'b2_delete(folder/a.txt, id_a_8380800000, (old version))', ], ), ], ) def test_newer_delete_old_versions(self, synchronizer_factory, src_type, expected): synchronizer = synchronizer_factory(keep_days_or_delete=KeepOrDeleteMode.DELETE) src = self.folder_factory(src_type, ('a.txt', [TODAY])) dst = self.b2_folder_factory(('a.txt', [TODAY - 1 * DAY, TODAY - 3 * DAY])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) # # src older than dst @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 200)']), ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)']), ], ) def test_older(self, synchronizer, apiver, src_type, dst_type, expected): src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type, ('a.txt', [200])) with pytest.raises(DestFileNewer) as excinfo: self.assert_folder_sync_actions(synchronizer, src, dst, expected) messages = defaultdict( lambda: 'source file is older than destination: %s://a.txt with a time of 100 ' 'cannot be synced to %s://a.txt with a time of 200, ' 'unless a valid newer_file_mode is provided', v0='source file is older than destination: %s://a.txt with a time of 100 ' 'cannot be synced to %s://a.txt with a time of 200, ' 'unless --skipNewer or --replaceNewer is provided', ) assert str(excinfo.value) == messages[apiver] % (src_type, dst_type) @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_older_skip(self, synchronizer_factory, src_type, dst_type): synchronizer = synchronizer_factory(newer_file_mode=NewerFileSyncMode.SKIP) src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type, ('a.txt', [200])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 100)']), ('b2', 'local', ['b2_download(folder/a.txt, id_a_100, /dir/a.txt, 100)']), ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_100, folder/a.txt, 100)']), ], ) def test_older_replace(self, synchronizer_factory, src_type, dst_type, expected): synchronizer = synchronizer_factory(newer_file_mode=NewerFileSyncMode.REPLACE) src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type, ('a.txt', [200])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,expected', [ ( 'local', [ 'b2_upload(/dir/a.txt, folder/a.txt, 100)', 'b2_delete(folder/a.txt, id_a_200, (old version))', ], ), ( 'b2', [ 'b2_copy(folder/a.txt, id_a_100, folder/a.txt, 100)', 'b2_delete(folder/a.txt, id_a_200, (old version))', ], ), ], ) def test_older_replace_delete(self, synchronizer_factory, src_type, expected): synchronizer = synchronizer_factory( newer_file_mode=NewerFileSyncMode.REPLACE, keep_days_or_delete=KeepOrDeleteMode.DELETE ) src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.b2_folder_factory(('a.txt', [200])) self.assert_folder_sync_actions(synchronizer, src, dst, expected) # # compareVersions option @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_compare_none_newer(self, synchronizer_factory, src_type, dst_type): synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.NONE) src = self.folder_factory(src_type, ('a.txt', [200])) dst = self.folder_factory(dst_type, ('a.txt', [100])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_compare_none_older(self, synchronizer_factory, src_type, dst_type): synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.NONE) src = self.folder_factory(src_type, ('a.txt', [100])) dst = self.folder_factory(dst_type, ('a.txt', [200])) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type,dst_type', [ ('local', 'b2'), ('b2', 'local'), ('b2', 'b2'), ], ) def test_compare_size_equal(self, synchronizer_factory, src_type, dst_type): synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.SIZE) src = self.folder_factory(src_type, ('a.txt', [200], 10)) dst = self.folder_factory(dst_type, ('a.txt', [100], 10)) self.assert_folder_sync_actions(synchronizer, src, dst, []) @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ('local', 'b2', ['b2_upload(/dir/a.txt, folder/a.txt, 200)']), ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), ('b2', 'b2', ['b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)']), ], ) def test_compare_size_not_equal(self, synchronizer_factory, src_type, dst_type, expected): synchronizer = synchronizer_factory(compare_version_mode=CompareVersionMode.SIZE) src = self.folder_factory(src_type, ('a.txt', [200], 11)) dst = self.folder_factory(dst_type, ('a.txt', [100], 10)) self.assert_folder_sync_actions(synchronizer, src, dst, expected) @pytest.mark.parametrize( 'src_type,dst_type,expected', [ ( 'local', 'b2', [ 'b2_upload(/dir/a.txt, folder/a.txt, 200)', 'b2_delete(folder/a.txt, id_a_100, (old version))', ], ), ('b2', 'local', ['b2_download(folder/a.txt, id_a_200, /dir/a.txt, 200)']), ( 'b2', 'b2', [ 'b2_copy(folder/a.txt, id_a_200, folder/a.txt, 200)', 'b2_delete(folder/a.txt, id_a_100, (old version))', ], ), ], ) def test_compare_size_not_equal_delete( self, synchronizer_factory, src_type, dst_type, expected ): synchronizer = synchronizer_factory( compare_version_mode=CompareVersionMode.SIZE, keep_days_or_delete=KeepOrDeleteMode.DELETE, ) src = self.folder_factory(src_type, ('a.txt', [200], 11)) dst = self.folder_factory(dst_type, ('a.txt', [100], 10)) self.assert_folder_sync_actions(synchronizer, src, dst, expected) # FIXME: rewrite this test to not use mock.call checks when all of Synchronizers tests are rewritten to test_bucket # style - i.e. with simulated api and fake files returned from methods. Then, checking EncryptionSetting used for # transmission will be done by the underlying simulator. def test_encryption_b2_to_local(self, synchronizer_factory, apiver): local = self.local_folder_factory() remote = self.b2_folder_factory(('directory/b.txt', [100])) synchronizer = synchronizer_factory() encryption = object() bucket = mock.MagicMock() provider = TstEncryptionSettingsProvider(encryption, encryption) download_action = next( iter( self._make_folder_sync_actions( synchronizer, remote, local, TODAY, self.reporter, encryption_settings_provider=provider, ) ) ) with mock.patch.object(B2DownloadAction, '_ensure_directory_existence'): try: download_action.do_action(bucket, self.reporter) except: # noqa: E722 pass assert bucket.mock_calls[0] == mock.call.download_file_by_id( 'id_d_100', progress_listener=mock.ANY, encryption=encryption ) if apiver in ['v0', 'v1']: file_version_kwarg = 'file_version_info' else: file_version_kwarg = 'file_version' provider.get_setting_for_download.assert_has_calls( [ mock.call( bucket=bucket, **{file_version_kwarg: mock.ANY}, ) ] ) # FIXME: rewrite this test to not use mock.call checks when all of Synchronizers tests are rewritten to test_bucket # style - i.e. with simulated api and fake files returned from methods. Then, checking EncryptionSetting used for # transmission will be done by the underlying simulator. def test_encryption_local_to_b2(self, synchronizer_factory): local = self.local_folder_factory(('directory/a.txt', [100])) remote = self.b2_folder_factory() synchronizer = synchronizer_factory() encryption = object() bucket = mock.MagicMock() provider = TstEncryptionSettingsProvider(encryption, encryption) upload_action = next( iter( self._make_folder_sync_actions( synchronizer, local, remote, TODAY, self.reporter, encryption_settings_provider=provider, ) ) ) with mock.patch.object(UploadSourceLocalFile, 'check_path_and_get_size'): try: upload_action.do_action(bucket, self.reporter) except: # noqa: E722 pass assert bucket.mock_calls == [ mock.call.concatenate( mock.ANY, 'folder/directory/a.txt', file_info={'src_last_modified_millis': '100'}, progress_listener=mock.ANY, encryption=encryption, large_file_sha1=None, ) ] assert provider.get_setting_for_upload.mock_calls == [ mock.call( bucket=bucket, b2_file_name='folder/directory/a.txt', file_info={'src_last_modified_millis': '100'}, length=10, ) ] # FIXME: rewrite this test to not use mock.call checks when all of Synchronizers tests are rewritten to test_bucket # style - i.e. with simulated api and fake files returned from methods. Then, checking EncryptionSetting used for # transmission will be done by the underlying simulator. def test_encryption_b2_to_b2(self, synchronizer_factory, apiver): src = self.b2_folder_factory(('directory/a.txt', [100])) dst = self.b2_folder_factory() synchronizer = synchronizer_factory() source_encryption = object() destination_encryption = object() bucket = mock.MagicMock() provider = TstEncryptionSettingsProvider(source_encryption, destination_encryption) copy_action = next( iter( self._make_folder_sync_actions( synchronizer, src, dst, TODAY, self.reporter, encryption_settings_provider=provider, ) ) ) copy_action.do_action(bucket, self.reporter) assert bucket.mock_calls == [ mock.call.copy( 'id_d_100', 'folder/directory/a.txt', length=10, source_content_type='text/plain', source_file_info={'in_b2': 'yes'}, progress_listener=mock.ANY, source_encryption=source_encryption, destination_encryption=destination_encryption, ) ] if apiver in ['v0', 'v1']: file_version_kwarg = 'source_file_version_info' additional_kwargs = {'target_file_info': None} else: file_version_kwarg = 'source_file_version' additional_kwargs = {} assert provider.get_source_setting_for_copy.mock_calls == [ mock.call( bucket=mock.ANY, **{file_version_kwarg: mock.ANY}, ) ] assert provider.get_destination_setting_for_copy.mock_calls == [ mock.call( bucket=mock.ANY, dest_b2_file_name='folder/directory/a.txt', **additional_kwargs, **{file_version_kwarg: mock.ANY}, ) ] def test_custom_sync_manager_policy(self, synchronizer_factory): class MySyncPolicyManager(SyncPolicyManager): def get_policy_class(self, sync_type, delete, keep_days): return UpPolicy synchronizer = synchronizer_factory( compare_version_mode=CompareVersionMode.SIZE, keep_days_or_delete=KeepOrDeleteMode.DELETE, sync_policy_manager=MySyncPolicyManager(), ) src = self.folder_factory('local', ('a.txt', [200], 11)) dst = self.folder_factory('b2', ('a.txt', [100], 10)) # normally_expected = [ # 'b2_upload(/dir/a.txt, folder/a.txt, 200)', # 'b2_delete(folder/a.txt, id_a_100, (old version))' # ] expected = ['b2_upload(/dir/a.txt, folder/a.txt, 200)'] self.assert_folder_sync_actions(synchronizer, src, dst, expected) # FIXME: rewrite this test to not use mock.call checks when all of Synchronizers tests are rewritten to test_bucket # style - i.e. with simulated api and fake files returned from methods. @pytest.mark.apiver(from_ver=2) @pytest.mark.parametrize( 'local_size,remote_size,local_sha1,local_partial_sha1,remote_sha1,should_be_incremental', [ (2000, 1000, 'ff' * 20, 'aa' * 20, 'aa' * 20, True), # incremental upload possible (2000, 999, 'ff' * 20, 'aa' * 20, 'aa' * 20, False), # uploaded part too small (2000, 1000, 'ff' * 20, 'aa' * 20, None, False), # remote sha unknown (2000, 1000, 'ff' * 20, 'aa' * 20, 'bb' * 20, False), # remote sha mismatch (2000, 3000, 'ff' * 20, 'aa' * 20, 'bb' * 20, False), # remote file bigger ], ) def test_incremental_upload( self, synchronizer_factory, local_size, remote_size, local_sha1, local_partial_sha1, remote_sha1, should_be_incremental, ): synchronizer = synchronizer_factory( upload_mode=UploadMode.INCREMENTAL, absolute_minimum_part_size=1000 ) src = self.folder_factory('local', ('a.txt', [200], local_size)) dst = self.folder_factory('b2', ('a.txt', [100], remote_size)) upload_action = next( iter(self._make_folder_sync_actions(synchronizer, src, dst, TODAY, self.reporter)) ) bucket = mock.MagicMock() def update_from_stream(self, limit=None): if limit is None: return local_sha1 elif limit == remote_size: return local_partial_sha1 else: assert False def check_path_and_get_size(self): self.content_length = local_size with ExitStack() as stack: patches = [ mock.patch.object( UploadSourceLocalFile, 'open', mock.mock_open(read_data='test-data') ), mock.patch.object(IncrementalHexDigester, 'update_from_stream', update_from_stream), mock.patch.object( UploadSourceLocalFile, 'check_path_and_get_size', check_path_and_get_size ), mock.patch.object( UploadSourceLocalFile, '_hex_sha1_of_file', return_value=local_sha1 ), mock.patch.object( UploadSourceLocalFileRange, 'check_path_and_get_size', check_path_and_get_size ), mock.patch.object(FileVersion, 'get_content_sha1', return_value=remote_sha1), ] for patch in patches: stack.enter_context(patch) upload_action.do_action(bucket, self.reporter) assert bucket.mock_calls == [ mock.call.concatenate( mock.ANY, 'folder/a.txt', file_info=mock.ANY, progress_listener=mock.ANY, encryption=None, large_file_sha1=local_sha1 if should_be_incremental else None, ) ] # In Python 3.7 unittest.mock.call doesn't have `args` properly defined. Instead we have to take 1st index. # TODO: use .args[0] instead of [1] when we drop Python 3.7 num_calls = len(bucket.mock_calls[0][1]) assert num_calls == 2 if should_be_incremental else 1, bucket.mock_calls[0] if should_be_incremental: # Order of indices: call index, pick arguments, pick first argument, first element of the first argument. assert isinstance(bucket.mock_calls[0][1][0][0], CopySource) def test_sync_lexicographical_order(self, synchronizer_factory): """ Test sync is successful when dir tree order is different from absolute path lexicographical order Regression test for #502 """ synchronizer = synchronizer_factory() files = [('a/foo', [100], 10), ('a b/bar', [100], 10)] src = self.folder_factory('local', *files) dst = self.folder_factory('b2') self.assert_folder_sync_actions( synchronizer, src, dst, [ 'b2_upload(/dir/a b/bar, folder/a b/bar, 100)', 'b2_upload(/dir/a/foo, folder/a/foo, 100)', ], ) dst_with_files = self.folder_factory('b2', *files) self.assert_folder_sync_actions(synchronizer, src, dst_with_files, []) class TstEncryptionSettingsProvider(AbstractSyncEncryptionSettingsProvider): def __init__(self, source_encryption_setting, destination_encryption_setting): self.get_setting_for_upload = mock.MagicMock( side_effect=lambda *a, **kw: destination_encryption_setting ) self.get_source_setting_for_copy = mock.MagicMock( side_effect=lambda *a, **kw: source_encryption_setting ) self.get_destination_setting_for_copy = mock.MagicMock( side_effect=lambda *a, **kw: destination_encryption_setting ) self.get_setting_for_download = mock.MagicMock( side_effect=lambda *a, **kw: source_encryption_setting ) def get_setting_for_upload(self, *a, **kw): """overwritten in __init__""" def get_source_setting_for_copy(self, *a, **kw): """overwritten in __init__""" def get_destination_setting_for_copy(self, *a, **kw): """overwritten in __init__""" def get_setting_for_download(self, *a, **kw): """overwritten in __init__""" b2-sdk-python-2.8.0/test/unit/sync/test_sync_report.py000066400000000000000000000032361474454370000230300ustar00rootroot00000000000000###################################################################### # # File: test/unit/sync/test_sync_report.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from unittest.mock import MagicMock import pytest from apiver_deps import SyncReport class TestSyncReport: def test_bad_terminal(self): stdout = MagicMock() stdout.write = MagicMock( side_effect=[ UnicodeEncodeError('codec', 'foo', 100, 105, 'artificial UnicodeEncodeError') ] + list(range(25)) ) sync_report = SyncReport(stdout, False) sync_report.print_completion('transferred: 123.txt') @pytest.mark.apiver(to_ver=1) def test_legacy_methods(self): stdout = MagicMock() sync_report = SyncReport(stdout, False) assert not sync_report.total_done assert not sync_report.local_done assert 0 == sync_report.total_count assert 0 == sync_report.local_file_count sync_report.local_done = True assert sync_report.local_done assert sync_report.total_done sync_report.local_file_count = 8 assert 8 == sync_report.local_file_count assert 8 == sync_report.total_count sync_report.update_local(7) assert 15 == sync_report.total_count assert 15 == sync_report.local_file_count sync_report = SyncReport(stdout, False) assert not sync_report.total_done sync_report.end_local() assert sync_report.total_done b2-sdk-python-2.8.0/test/unit/test_base.py000066400000000000000000000034501474454370000204150ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_base.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import re import unittest from contextlib import contextmanager import apiver_deps from apiver_deps import B2Api from b2sdk.v2 import FullApplicationKey class TestBase(unittest.TestCase): @contextmanager def assertRaises(self, exc, msg=None): try: yield except exc as e: if msg is not None: if msg != str(e): assert False, f"expected message '{msg}', but got '{str(e)}'" else: assert False, f'should have thrown {exc}' @contextmanager def assertRaisesRegexp(self, expected_exception, expected_regexp): try: yield except expected_exception as e: if not re.search(expected_regexp, str(e)): assert False, f"expected message '{expected_regexp}', but got '{str(e)}'" else: assert False, f'should have thrown {expected_exception}' def create_key( api: B2Api, capabilities: list[str], key_name: str, valid_duration_seconds: int | None = None, bucket_id: str | None = None, name_prefix: str | None = None, ) -> FullApplicationKey: """apiver-agnostic B2Api.create_key""" result = api.create_key( capabilities=capabilities, key_name=key_name, valid_duration_seconds=valid_duration_seconds, bucket_id=bucket_id, name_prefix=name_prefix, ) if apiver_deps.V <= 1: return FullApplicationKey.from_create_response(result) return result b2-sdk-python-2.8.0/test/unit/test_cache.py000066400000000000000000000051201474454370000205420ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_cache.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from dataclasses import dataclass import pytest from apiver_deps import AuthInfoCache, DummyCache, InMemoryAccountInfo, InMemoryCache from pytest_lazy_fixtures import lf @pytest.fixture def dummy_cache(): return DummyCache() @pytest.fixture def in_memory_cache(): return InMemoryCache() @pytest.fixture def auth_info_cache(): return AuthInfoCache(InMemoryAccountInfo()) @pytest.fixture(scope='function', params=[lf('in_memory_cache'), lf('auth_info_cache')]) def cache(request): return request.param @dataclass class DummyBucket: name: str id_: str @pytest.fixture def buckets(): class InfBuckets(list): def __getitem__(self, item: int): self.extend(DummyBucket(f'bucket{i}', f'ID-{i}') for i in range(len(self), item + 1)) return super().__getitem__(item) return InfBuckets() class TestCache: def test_save_bucket(self, cache, buckets): cache.save_bucket(buckets[0]) def test_get_bucket_id_or_none_from_bucket_name(self, cache, buckets): assert cache.get_bucket_id_or_none_from_bucket_name('bucket0') is None cache.save_bucket(buckets[0]) assert cache.get_bucket_id_or_none_from_bucket_name('bucket0') == 'ID-0' def test_get_bucket_name_or_none_from_bucket_id(self, cache, buckets): assert cache.get_bucket_name_or_none_from_bucket_id('ID-0') is None cache.save_bucket(buckets[0]) assert cache.get_bucket_name_or_none_from_bucket_id('ID-0') == 'bucket0' @pytest.mark.apiver(from_ver=3) def test_list_bucket_names_ids(self, cache, buckets): assert cache.list_bucket_names_ids() == [] for i in range(2): cache.save_bucket(buckets[i]) assert cache.list_bucket_names_ids() == [('bucket0', 'ID-0'), ('bucket1', 'ID-1')] def test_set_bucket_name_cache(self, cache, buckets): cache.set_bucket_name_cache([buckets[i] for i in range(2, 4)]) assert cache.get_bucket_id_or_none_from_bucket_name('bucket1') is None assert cache.get_bucket_id_or_none_from_bucket_name('bucket2') == 'ID-2' cache.set_bucket_name_cache([buckets[1]]) assert cache.get_bucket_id_or_none_from_bucket_name('bucket1') == 'ID-1' assert cache.get_bucket_id_or_none_from_bucket_name('bucket2') is None b2-sdk-python-2.8.0/test/unit/test_exception.py000066400000000000000000000145731474454370000215110ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_exception.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from apiver_deps_exception import ( AlreadyFailed, B2Error, BadJson, BadUploadUrl, BucketIdNotFound, CapExceeded, Conflict, DuplicateBucketName, EmailNotVerified, FileAlreadyHidden, FileNotPresent, InvalidAuthToken, MissingPart, NoPaymentHistory, PartSha1Mismatch, ServiceError, StorageCapExceeded, TooManyRequests, TransactionCapExceeded, Unauthorized, UnknownError, interpret_b2_error, ) from b2sdk._internal.exception import ResourceNotFound class TestB2Error: def test_plain_ascii(self): assert 'message' == str(B2Error('message')) def test_unicode(self): assert '\u81ea\u7531' == str(B2Error('\u81ea\u7531')) class TestExceptions: def test_bad_upload_url_exception(self): try: raise BadUploadUrl('foo') except BadUploadUrl as e: assert not e.should_retry_http() assert e.should_retry_upload() assert str(e) == 'Bad upload url: foo', str(e) def test_already_failed_exception(self): try: raise AlreadyFailed('foo') except AlreadyFailed as e: assert str(e) == 'Already failed: foo', str(e) @pytest.mark.apiver(to_ver=1) def test_command_error(self): from apiver_deps_exception import CommandError try: raise CommandError('foo') except CommandError as e: assert str(e) == 'foo', str(e) class TestInterpretError: def test_file_already_hidden(self): self._check_one(FileAlreadyHidden, 400, 'already_hidden', '', {}) assert 'File already hidden: file.txt' == str( interpret_b2_error(400, 'already_hidden', '', {}, {'fileName': 'file.txt'}) ) def test_bad_json(self): self._check_one(BadJson, 400, 'bad_json', '', {}) def test_file_not_present(self): self._check_one(FileNotPresent, 400, 'no_such_file', '', {}) self._check_one(FileNotPresent, 400, 'file_not_present', '', {}) self._check_one(FileNotPresent, 404, 'not_found', '', {}) assert 'File not present: file.txt' == str( interpret_b2_error(404, 'not_found', '', {}, {'fileName': 'file.txt'}) ) assert 'File not present: 01010101' == str( interpret_b2_error(404, 'not_found', '', {}, {'fileId': '01010101'}) ) def test_file_or_bucket_not_present(self): self._check_one(ResourceNotFound, 404, None, None, {}) assert 'No such file, bucket, or endpoint: ' == str(interpret_b2_error(404, None, None, {})) def test_duplicate_bucket_name(self): self._check_one(DuplicateBucketName, 400, 'duplicate_bucket_name', '', {}) assert 'Bucket name is already in use: my-bucket' == str( interpret_b2_error(400, 'duplicate_bucket_name', '', {}, {'bucketName': 'my-bucket'}) ) def test_missing_part(self): self._check_one(MissingPart, 400, 'missing_part', '', {}) assert 'Part number has not been uploaded: my-file-id' == str( interpret_b2_error(400, 'missing_part', '', {}, {'fileId': 'my-file-id'}) ) def test_part_sha1_mismatch(self): self._check_one(PartSha1Mismatch, 400, 'part_sha1_mismatch', '', {}) assert 'Part number my-file-id has wrong SHA1' == str( interpret_b2_error(400, 'part_sha1_mismatch', '', {}, {'fileId': 'my-file-id'}) ) def test_unauthorized(self): self._check_one(Unauthorized, 401, '', '', {}) def test_invalid_auth_token(self): self._check_one(InvalidAuthToken, 401, 'bad_auth_token', '', {}) self._check_one(InvalidAuthToken, 401, 'expired_auth_token', '', {}) def test_storage_cap_exceeded(self): self._check_one((CapExceeded, StorageCapExceeded), 403, 'storage_cap_exceeded', '', {}) def test_transaction_cap_exceeded(self): self._check_one( (CapExceeded, TransactionCapExceeded), 403, 'transaction_cap_exceeded', '', {} ) def test_conflict(self): self._check_one(Conflict, 409, '', '', {}) def test_too_many_requests_with_retry_after_header(self): retry_after = 200 error = self._check_one( TooManyRequests, 429, '', '', {'retry-after': retry_after}, ) assert error.retry_after_seconds == retry_after def test_too_many_requests_without_retry_after_header(self): error = self._check_one(TooManyRequests, 429, '', '', {}) assert error.retry_after_seconds is None @pytest.mark.apiver( from_ver=3 ) # previous apivers throw this as well, but BucketIdNotFound is a different class in them def test_bad_bucket_id(self): error = self._check_one( BucketIdNotFound, 400, 'bad_bucket_id', '', {}, {'bucketId': '1001'} ) assert error.bucket_id == '1001' def test_service_error(self): error = interpret_b2_error(500, 'code', 'message', {}) assert isinstance(error, ServiceError) assert '500 code message' == str(error) def test_unknown_error(self): error = interpret_b2_error(499, 'code', 'message', {}) assert isinstance(error, UnknownError) assert 'Unknown error: 499 code message' == str(error) @classmethod def _check_one( cls, expected_class, status, code, message, response_headers, post_params=None, ): actual_exception = interpret_b2_error(status, code, message, response_headers, post_params) assert isinstance(actual_exception, expected_class) return actual_exception @pytest.mark.parametrize( 'status, code, expected_exception_cls', [ (401, 'email_not_verified', EmailNotVerified), (401, 'no_payment_history', NoPaymentHistory), ], ) def test_simple_error_handlers(self, status, code, expected_exception_cls): error = interpret_b2_error(status, code, '', {}) assert isinstance(error, expected_exception_cls) assert error.code == code b2-sdk-python-2.8.0/test/unit/test_included_modules.py000066400000000000000000000012311474454370000230150ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_included_modules.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pathlib from b2sdk._internal import requests from b2sdk._internal.requests.included_source_meta import included_source_meta def test_requests_notice_file(): with (pathlib.Path(requests.__file__).parent / 'NOTICE').open('r') as notice_file: assert notice_file.read() == included_source_meta.files['NOTICE'] b2-sdk-python-2.8.0/test/unit/test_progress.py000066400000000000000000000021351474454370000213460ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_progress.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from apiver_deps import TqdmProgressListener, make_progress_listener @pytest.mark.parametrize( 'tqdm_available, quiet, expected_listener', [ (True, False, 'TqdmProgressListener'), (False, False, 'SimpleProgressListener'), (False, True, 'DoNothingProgressListener'), ], ) def test_make_progress_listener(tqdm_available, quiet, expected_listener, monkeypatch): if not tqdm_available: monkeypatch.setattr('b2sdk._internal.progress.tqdm', None) assert make_progress_listener('description', quiet).__class__.__name__ == expected_listener def test_tqdm_progress_listener__without_tqdm_module(monkeypatch): monkeypatch.setattr('b2sdk._internal.progress.tqdm', None) with pytest.raises(ModuleNotFoundError): TqdmProgressListener('description') b2-sdk-python-2.8.0/test/unit/test_raw_simulator.py000066400000000000000000000107201474454370000223710ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_raw_simulator.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import hashlib from unittest.mock import Mock import pytest from b2sdk import _v3 as v3 from test.helpers import patch_bind_params @pytest.fixture def dummy_bucket_simulator(apiver_module): return apiver_module.BucketSimulator( Mock(spec=apiver_module.B2Api), 'account_id', 'bucket_id', 'bucket_name', 'allPublic' ) @pytest.fixture def dummy_raw_simulator(apiver_module): return apiver_module.RawSimulator() @pytest.fixture def file_sim(apiver_module, dummy_bucket_simulator, file_info): data = b'dummy' return apiver_module.FileSimulator( account_id='account_id', bucket=dummy_bucket_simulator, file_id='dummy-id', action='upload', name='dummy.txt', content_type='text/plain', content_sha1=hashlib.sha1(data).hexdigest(), file_info=file_info, data_bytes=data, upload_timestamp=0, server_side_encryption=apiver_module.EncryptionSetting( mode=apiver_module.EncryptionMode.SSE_C, algorithm=apiver_module.EncryptionAlgorithm.AES256, key=apiver_module.EncryptionKey(key_id=None, secret=b'test'), ), ) def test_file_sim__as_download_headers(file_sim): assert file_sim.as_download_headers() == { 'content-length': '5', 'content-type': 'text/plain', 'x-bz-content-sha1': '829c3804401b0727f70f73d4415e162400cbe57b', 'x-bz-upload-timestamp': '0', 'x-bz-file-id': 'dummy-id', 'x-bz-file-name': 'dummy.txt', 'X-Bz-Info-key': 'value', 'X-Bz-Server-Side-Encryption-Customer-Algorithm': 'AES256', 'X-Bz-Server-Side-Encryption-Customer-Key-Md5': 'CY9rzUYh03PK3k6DJie09g==', } @pytest.mark.apiver(to_ver=2) def test_bucket_simulator__upload_file__supports_file_infos( apiver_module, dummy_bucket_simulator, file_info ): """Test v2.BucketSimulator.upload_file support of deprecated file_infos param""" with patch_bind_params(v3.BucketSimulator, 'upload_file') as mock_method, pytest.warns( DeprecationWarning, match=r'deprecated argument' ): assert dummy_bucket_simulator.upload_file( 'upload_id', 'upload_auth_token', 'file_name', 123, # content_length 'content_type', 'content_sha1', file_infos=file_info, data_stream='data_stream', ) assert mock_method.get_bound_call_args()['file_info'] == file_info assert 'file_infos' not in mock_method.call_args[1] @pytest.mark.apiver(to_ver=2) def test_raw_simulator__get_upload_file_headers__supports_file_infos(apiver_module, file_info): """Test v2.RawSimulator.get_upload_file_headers support of deprecated file_infos param""" with patch_bind_params(v3.RawSimulator, 'get_upload_file_headers') as mock_method, pytest.warns( DeprecationWarning, match=r'deprecated argument' ): apiver_module.RawSimulator.get_upload_file_headers( upload_auth_token='upload_auth_token', file_name='file_name', content_type='content_type', content_length=123, content_sha1='content_sha1', server_side_encryption=None, file_retention=None, legal_hold=None, file_infos=file_info, ) assert mock_method.get_bound_call_args()['file_info'] == file_info assert 'file_infos' not in mock_method.call_args[1] @pytest.mark.apiver(to_ver=2) def test_raw_simulator__upload_file__supports_file_infos(dummy_raw_simulator, file_info): """Test v2.RawSimulator.upload_file support of deprecated file_infos param""" with patch_bind_params(v3.RawSimulator, 'upload_file') as mock_method, pytest.warns( DeprecationWarning, match=r'deprecated argument' ): dummy_raw_simulator.upload_file( 'upload_url', 'upload_auth_token', 'file_name', 123, 'content_type', 'content_sha1', file_infos=file_info, data_stream='data_stream', ) assert mock_method.get_bound_call_args()['file_info'] == file_info assert 'file_infos' not in mock_method.call_args[1] b2-sdk-python-2.8.0/test/unit/test_session.py000066400000000000000000000056421474454370000211730ustar00rootroot00000000000000###################################################################### # # File: test/unit/test_session.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from unittest import mock from apiver_deps import AuthInfoCache, B2Session, DummyCache, InMemoryAccountInfo from .account_info.fixtures import * # noqa from .fixtures import * # noqa class TestAuthorizeAccount: @pytest.fixture(autouse=True) def setup(self, b2_session): self.b2_session = b2_session @pytest.mark.parametrize( 'authorize_call_kwargs', [ pytest.param( dict( account_id=mock.ANY, auth_token=mock.ANY, api_url=mock.ANY, download_url=mock.ANY, recommended_part_size=mock.ANY, absolute_minimum_part_size=mock.ANY, application_key='456', realm='dev', s3_api_url=mock.ANY, allowed=mock.ANY, application_key_id='123', ), marks=pytest.mark.apiver(from_ver=2), ), pytest.param( dict( account_id=mock.ANY, auth_token=mock.ANY, api_url=mock.ANY, download_url=mock.ANY, minimum_part_size=mock.ANY, application_key='456', realm='dev', s3_api_url=mock.ANY, allowed=mock.ANY, application_key_id='123', ), marks=pytest.mark.apiver(to_ver=1), ), ], ) def test_simple_authorization(self, authorize_call_kwargs): self.b2_session.authorize_account('dev', '123', '456') self.b2_session.raw_api.authorize_account.assert_called_once_with( 'http://api.backblazeb2.xyz:8180', '123', '456' ) assert self.b2_session.cache.clear.called is False self.b2_session.account_info.set_auth_data.assert_called_once_with(**authorize_call_kwargs) def test_clear_cache(self): self.b2_session.account_info.is_same_account.return_value = False self.b2_session.authorize_account('dev', '123', '456') assert self.b2_session.cache.clear.called is True def test_session__with_in_memory_account_info(apiver_int): memory_info = InMemoryAccountInfo() b2_session = B2Session( account_info=memory_info, ) assert b2_session.account_info is memory_info if apiver_int < 3: assert isinstance(b2_session.cache, DummyCache) else: assert isinstance(b2_session.cache, AuthInfoCache) assert b2_session.cache.info is memory_info b2-sdk-python-2.8.0/test/unit/utils/000077500000000000000000000000001474454370000172305ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/utils/__init__.py000066400000000000000000000005111474454370000213360ustar00rootroot00000000000000###################################################################### # # File: test/unit/utils/__init__.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/utils/test_docs.py000066400000000000000000000023631474454370000215750ustar00rootroot00000000000000###################################################################### # # File: test/unit/utils/test_docs.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from b2sdk._internal.raw_api import AbstractRawApi, LifecycleRule from b2sdk._internal.utils.docs import MissingDocURL, ensure_b2sdk_doc_urls, get_b2sdk_doc_urls def test_b2sdk_doc_urls(): @ensure_b2sdk_doc_urls class MyCustomClass: """ This is a custom class with `Documentation URL`_. .. _Documentation URL: https://example.com """ def test_b2sdk_doc_urls__no_urls_error(): with pytest.raises(MissingDocURL): @ensure_b2sdk_doc_urls class MyCustomClass: pass @pytest.mark.parametrize( 'type_,expected', [ (AbstractRawApi, {}), ( LifecycleRule, { 'B2 Cloud Storage Lifecycle Rules': 'https://www.backblaze.com/docs/cloud-storage-lifecycle-rules', }, ), ], ) def test_get_b2sdk_doc_urls(type_, expected): assert get_b2sdk_doc_urls(type_) == expected b2-sdk-python-2.8.0/test/unit/utils/test_escape.py000066400000000000000000000043131474454370000221020ustar00rootroot00000000000000###################################################################### # # File: test/unit/utils/test_escape.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest from b2sdk._internal.utils.escape import ( escape_control_chars, substitute_control_chars, unprintable_to_hex, ) @pytest.mark.parametrize( ( 'input_', 'expected_unprintable_to_hex', 'expected_escape_control_chars', 'expected_substitute_control_chars', ), [ ('', '', '', ('', False)), (' abc-z', ' abc-z', "' abc-z'", (' abc-z', False)), ('a\x7fb', 'a\\x7fb', "'a\\x7fb'", ('a�b', True)), ('a\x00b a\x9fb ', 'a\\x00b a\\x9fb ', "'a\\x00b a\\x9fb '", ('a�b a�b ', True)), ('a\x7fb\nc', 'a\\x7fb\nc', "'a\\x7fb\nc'", ('a�b\nc', True)), ('\x9bT\x9bEtest', '\\x9bT\\x9bEtest', "'\\x9bT\\x9bEtest'", ('�T�Etest', True)), ( '\x1b[32mC\x1b[33mC\x1b[34mI', '\\x1b[32mC\\x1b[33mC\\x1b[34mI', "'\\x1b[32mC\\x1b[33mC\\x1b[34mI'", ('�[32mC�[33mC�[34mI', True), ), ], ) def test_unprintable_to_hex( input_, expected_unprintable_to_hex, expected_escape_control_chars, expected_substitute_control_chars, ): assert unprintable_to_hex(input_) == expected_unprintable_to_hex assert escape_control_chars(input_) == expected_escape_control_chars assert substitute_control_chars(input_) == expected_substitute_control_chars def test_unprintable_to_hex__none(): """ Test that unprintable_to_hex handles None. This was unintentionally supported and is only kept for compatibility. """ assert unprintable_to_hex(None) is None # type: ignore def test_escape_control_chars__none(): """ Test that escape_control_chars handles None. This was unintentionally supported and is only kept for compatibility. """ assert escape_control_chars(None) is None # type: ignore def test_substitute_control_chars__none(): with pytest.raises(TypeError): substitute_control_chars(None) # type: ignore b2-sdk-python-2.8.0/test/unit/utils/test_filesystem.py000066400000000000000000000030531474454370000230260ustar00rootroot00000000000000###################################################################### # # File: test/unit/utils/test_filesystem.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os import pathlib import platform import pytest from apiver_deps import ( STDOUT_FILEPATH, points_to_fifo, points_to_stdout, ) EXPECTED_STDOUT_PATH = pathlib.Path('CON' if platform.system() == 'Windows' else '/dev/stdout') class TestPointsToFifo: @pytest.mark.skipif(platform.system() == 'Windows', reason='no os.mkfifo() on Windows') def test_fifo_path(self, tmp_path): fifo_path = tmp_path / 'fifo' os.mkfifo(fifo_path) assert points_to_fifo(fifo_path) is True def test_non_fifo_path(self, tmp_path): path = tmp_path / 'subdir' path.mkdir(parents=True) assert points_to_fifo(path) is False def test_non_existent_path(self, tmp_path): path = tmp_path / 'file.txt' assert points_to_fifo(path) is False class TestPointsToStdout: def test_stdout_path(self): assert points_to_stdout(EXPECTED_STDOUT_PATH) is True assert points_to_stdout(STDOUT_FILEPATH) is True def test_non_stdout_path(self, tmp_path): path = tmp_path / 'file.txt' path.touch() assert points_to_stdout(path) is False def test_non_existent_stdout_path(self, tmp_path): path = tmp_path / 'file.txt' assert points_to_stdout(path) is False b2-sdk-python-2.8.0/test/unit/utils/test_incremental_hex_digester.py000066400000000000000000000052211474454370000256740ustar00rootroot00000000000000###################################################################### # # File: test/unit/utils/test_incremental_hex_digester.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import hashlib import io from b2sdk._internal.utils import ( IncrementalHexDigester, Sha1HexDigest, ) from test.unit.test_base import TestBase class TestIncrementalHexDigester(TestBase): BLOCK_SIZE = 4 def _get_sha1(self, input_data: bytes) -> Sha1HexDigest: return Sha1HexDigest(hashlib.sha1(input_data).hexdigest()) def _get_digester(self, stream: io.IOBase) -> IncrementalHexDigester: return IncrementalHexDigester(stream, block_size=self.BLOCK_SIZE) def test_limited_read(self): limit = self.BLOCK_SIZE * 10 input_data = b'1' * limit * 2 stream = io.BytesIO(input_data) expected_sha1 = self._get_sha1(input_data[:limit]) result_sha1 = self._get_digester(stream).update_from_stream(limit) self.assertEqual(expected_sha1, result_sha1) self.assertEqual(limit, stream.tell()) def test_limited_read__stream_smaller_than_block_size(self): limit = self.BLOCK_SIZE * 99 input_data = b'1' * (self.BLOCK_SIZE - 1) stream = io.BytesIO(input_data) expected_sha1 = self._get_sha1(input_data) result_sha1 = self._get_digester(stream).update_from_stream(limit) self.assertEqual(expected_sha1, result_sha1) self.assertEqual(len(input_data), stream.tell()) def test_unlimited_read(self): input_data = b'1' * self.BLOCK_SIZE * 10 stream = io.BytesIO(input_data) expected_sha1 = self._get_sha1(input_data) result_sha1 = self._get_digester(stream).update_from_stream() self.assertEqual(expected_sha1, result_sha1) self.assertEqual(len(input_data), stream.tell()) def test_limited_and_unlimited_read(self): blocks_count = 5 limit = self.BLOCK_SIZE * 5 input_data = b'1' * limit * blocks_count stream = io.BytesIO(input_data) digester = self._get_digester(stream) for idx in range(blocks_count - 1): expected_sha1_part = self._get_sha1(input_data[: limit * (idx + 1)]) result_sha1_part = digester.update_from_stream(limit) self.assertEqual(expected_sha1_part, result_sha1_part) expected_sha1_whole = self._get_sha1(input_data) result_sha1_whole = digester.update_from_stream() self.assertEqual(expected_sha1_whole, result_sha1_whole) b2-sdk-python-2.8.0/test/unit/utils/test_range_.py000066400000000000000000000041501474454370000220740ustar00rootroot00000000000000###################################################################### # # File: test/unit/utils/test_range_.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import pytest def test_range_initialization(apiver_module): r = apiver_module.Range(0, 10) assert r.start == 0 assert r.end == 10 def test_range_eq(apiver_module): r = apiver_module.Range(5, 10) assert r == apiver_module.Range(5, 10) assert r != apiver_module.Range(5, 11) assert r != apiver_module.Range(6, 10) def test_range_initialization_invalid(apiver_module): with pytest.raises(AssertionError): apiver_module.Range(10, 0) def test_range_from_header(apiver_module): r = apiver_module.Range.from_header('bytes=0-11') assert r.start == 0 assert r.end == 11 @pytest.mark.parametrize( 'raw_range_header, start, end, total_length', [ ('bytes 0-11', 0, 11, None), ('bytes 1-11/*', 1, 11, None), ('bytes 10-110/200', 10, 110, 200), ], ) def test_range_from_header_with_size(apiver_module, raw_range_header, start, end, total_length): r, length = apiver_module.Range.from_header_with_size(raw_range_header) assert r.start == start assert r.end == end assert length == total_length def test_range_size(apiver_module): r = apiver_module.Range(0, 10) assert r.size() == 11 def test_range_subrange(apiver_module): r = apiver_module.Range(1, 10) assert r.subrange(0, 9) == apiver_module.Range(1, 10) assert r.subrange(2, 5) == apiver_module.Range(3, 6) def test_range_subrange_invalid(apiver_module): r = apiver_module.Range(0, 10) with pytest.raises(AssertionError): r.subrange(5, 15) def test_range_as_tuple(apiver_module): r = apiver_module.Range(0, 10) assert r.as_tuple() == (0, 10) def test_range_repr(apiver_module): r = apiver_module.Range(0, 10) assert repr(r) == 'Range(0, 10)' def test_empty_range(apiver_module): r = apiver_module.EMPTY_RANGE assert r.size() == 0 b2-sdk-python-2.8.0/test/unit/utils/test_thread_pool.py000066400000000000000000000022301474454370000231360ustar00rootroot00000000000000###################################################################### # # File: test/unit/utils/test_thread_pool.py # # Copyright 2024 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from concurrent.futures import Future import pytest from b2sdk._internal.utils.thread_pool import LazyThreadPool class TestLazyThreadPool: @pytest.fixture def thread_pool(self): return LazyThreadPool() def test_submit(self, thread_pool): future = thread_pool.submit(sum, (1, 2)) assert isinstance(future, Future) assert future.result() == 3 def test_set_size(self, thread_pool): thread_pool.set_size(10) assert thread_pool.get_size() == 10 def test_get_size(self, thread_pool): assert thread_pool.get_size() > 0 def test_set_size__after_submit(self, thread_pool): future = thread_pool.submit(sum, (1, 2)) thread_pool.set_size(7) assert thread_pool.get_size() == 7 assert future.result() == 3 assert thread_pool.submit(sum, (1,)).result() == 1 b2-sdk-python-2.8.0/test/unit/v0/000077500000000000000000000000001474454370000164155ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/v0/__init__.py000066400000000000000000000005061474454370000205270ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/v0/apiver/000077500000000000000000000000001474454370000177035ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/v0/apiver/__init__.py000066400000000000000000000006651474454370000220230ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/apiver/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details b2-sdk-python-2.8.0/test/unit/v0/apiver/apiver_deps.py000066400000000000000000000005671474454370000225660ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/apiver/apiver_deps.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk.v0 import * # noqa V = 0 b2-sdk-python-2.8.0/test/unit/v0/apiver/apiver_deps_exception.py000066400000000000000000000006041474454370000246340ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/apiver/apiver_deps_exception.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk.v0.exception import * # noqa b2-sdk-python-2.8.0/test/unit/v0/deps.py000066400000000000000000000011731474454370000177240ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/deps.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # TODO: This module is used in old-style unit tests, written separately for v0 and v1. # It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details from apiver_deps import * b2-sdk-python-2.8.0/test/unit/v0/deps_exception.py000066400000000000000000000012171474454370000220010ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/deps_exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # TODO: This module is used in old-style unit tests, written separately for v0 and v1. # It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details from apiver_deps_exception import * b2-sdk-python-2.8.0/test/unit/v0/test_bounded_queue_executor.py000066400000000000000000000040201474454370000245640ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_bounded_queue_executor.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import concurrent.futures as futures import time from ..test_base import TestBase from .deps import BoundedQueueExecutor class TestBoundedQueueExecutor(TestBase): def setUp(self): unbounded_executor = futures.ThreadPoolExecutor(max_workers=1) self.executor = BoundedQueueExecutor(unbounded_executor, 1) def tearDown(self): self.executor.shutdown() def test_return_future(self): future_1 = self.executor.submit(lambda: 1) print(future_1) self.assertEqual(1, future_1.result()) def test_blocking(self): # This doesn't actually test that it waits, but it does exercise the code. # Make some futures using a function that takes a little time. def sleep_and_return_fcn(n): def fcn(): time.sleep(0.01) return n return fcn futures = [self.executor.submit(sleep_and_return_fcn(i)) for i in range(10)] # Check the answers answers = list(map(lambda f: f.result(), futures)) self.assertEqual(list(range(10)), answers) def test_no_exceptions(self): f = self.executor.submit(lambda: 1) self.executor.shutdown() self.assertEqual(0, self.executor.get_num_exceptions()) self.assertTrue(f.exception() is None) def test_two_exceptions(self): def thrower(): raise Exception('test_exception') f1 = self.executor.submit(thrower) f2 = self.executor.submit(thrower) self.executor.shutdown() self.assertEqual(2, self.executor.get_num_exceptions()) self.assertFalse(f1.exception() is None) self.assertEqual('test_exception', str(f2.exception())) b2-sdk-python-2.8.0/test/unit/v0/test_bucket.py000066400000000000000000001427571474454370000213230ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_bucket.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import os import platform import tempfile import unittest.mock as mock from io import BytesIO import pytest from ..test_base import TestBase from .deps import ( NO_RETENTION_FILE_SETTING, SSE_B2_AES, SSE_NONE, AbstractProgressListener, B2Api, BucketSimulator, CopySource, DownloadDestBytes, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, EncryptionSettingFactory, FakeResponse, FileRetentionSetting, FileSimulator, FileVersionInfo, LargeFileUploadState, LegalHold, MetadataDirectiveMode, ParallelDownloader, Part, PreSeekedDownloadDest, RawSimulator, RetentionMode, SimpleDownloader, StubAccountInfo, UploadSourceBytes, UploadSourceLocalFile, WriteIntent, hex_sha1_of_bytes, ) from .deps_exception import ( AlreadyFailed, B2Error, InvalidAuthToken, InvalidMetadataDirective, InvalidRange, InvalidUploadSource, MaxRetriesExceeded, SSECKeyError, UnsatisfiableRange, ) SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_key', key_id='some-id'), ) SSE_C_AES_NO_SECRET = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=None, key_id='some-id'), ) SSE_C_AES_2 = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_other_key', key_id='some-id-2'), ) SSE_C_AES_2_NO_SECRET = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=None, key_id='some-id-2'), ) SSE_C_AES_FROM_SERVER = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(key_id=None, secret=None), ) def write_file(path, data): with open(path, 'wb') as f: f.write(data) class StubProgressListener(AbstractProgressListener): """ Implementation of a progress listener that remembers what calls were made, and returns them as a short string to use in unit tests. For a total byte count of 100, and updates at 33 and 66, the returned string looks like: "100: 33 66" """ def __init__(self): self.total = None self.history = [] self.last_byte_count = 0 def get_history(self): return ' '.join(self.history) def set_total_bytes(self, total_byte_count): assert total_byte_count is not None assert self.total is None, 'set_total_bytes called twice' self.total = total_byte_count assert len(self.history) == 0, self.history self.history.append('%d:' % (total_byte_count,)) def bytes_completed(self, byte_count): self.last_byte_count = byte_count self.history.append(str(byte_count)) def is_valid(self, **kwargs): valid, _ = self.is_valid_reason(**kwargs) return valid def is_valid_reason(self, check_progress=True, check_monotonic_progress=False): progress_end = -1 if self.history[progress_end] == 'closed': progress_end = -2 # self.total != self.last_byte_count may be a consequence of non-monotonic # progress, so we want to check this first if check_monotonic_progress: prev = 0 for val in map(int, self.history[1:progress_end]): if val < prev: return False, 'non-monotonic progress' prev = val if self.total != self.last_byte_count: return False, 'total different than last_byte_count' if check_progress and len(self.history[1:progress_end]) < 2: return False, 'progress in history has less than 2 entries' return True, '' def close(self): self.history.append('closed') class CanRetry(B2Error): """ An exception that can be retryable, or not. """ def __init__(self, can_retry): super().__init__(None, None, None, None, None) self.can_retry = can_retry def should_retry_upload(self): return self.can_retry class TestCaseWithBucket(TestBase): RAW_SIMULATOR_CLASS = RawSimulator def setUp(self): self.bucket_name = 'my-bucket' self.simulator = self.RAW_SIMULATOR_CLASS() self.account_info = StubAccountInfo() self.api = B2Api(self.account_info, raw_api=self.simulator) (self.account_id, self.master_key) = self.simulator.create_account() self.api.authorize_account('production', self.account_id, self.master_key) self.api_url = self.account_info.get_api_url() self.account_auth_token = self.account_info.get_account_auth_token() self.bucket = self.api.create_bucket('my-bucket', 'allPublic') self.bucket_id = self.bucket.id_ def assertBucketContents(self, expected, *args, **kwargs): """ *args and **kwargs are passed to self.bucket.ls() """ actual = [ (info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket.ls(*args, **kwargs) ] self.assertEqual(expected, actual) def _make_data(self, approximate_length): """ Generate a sequence of bytes to use in testing an upload. Don't repeat a short pattern, so we're sure that the different parts of a large file are actually different. Returns bytes. """ fragments = [] so_far = 0 while so_far < approximate_length: fragment = ('%d:' % so_far).encode('utf-8') so_far += len(fragment) fragments.append(fragment) return b''.join(fragments) class TestReauthorization(TestCaseWithBucket): def testCreateBucket(self): class InvalidAuthTokenWrapper: def __init__(self, original_function): self.__original_function = original_function self.__name__ = original_function.__name__ self.__called = False def __call__(self, *args, **kwargs): if self.__called: return self.__original_function(*args, **kwargs) self.__called = True raise InvalidAuthToken('message', 401) self.simulator.create_bucket = InvalidAuthTokenWrapper(self.simulator.create_bucket) self.bucket = self.api.create_bucket('your-bucket', 'allPublic') class TestListParts(TestCaseWithBucket): def testEmpty(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) self.assertEqual([], list(self.bucket.list_parts(file1.file_id, batch_size=1))) def testThree(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) content = b'hello world' content_sha1 = hex_sha1_of_bytes(content) large_file_upload_state = mock.MagicMock() large_file_upload_state.has_error.return_value = False self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 1, large_file_upload_state ).result() self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 2, large_file_upload_state ).result() self.api.services.upload_manager.upload_part( self.bucket_id, file1.file_id, UploadSourceBytes(content), 3, large_file_upload_state ).result() expected_parts = [ Part('9999', 1, 11, content_sha1), Part('9999', 2, 11, content_sha1), Part('9999', 3, 11, content_sha1), ] self.assertEqual(expected_parts, list(self.bucket.list_parts(file1.file_id, batch_size=1))) class TestUploadPart(TestCaseWithBucket): def test_error_in_state(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) content = b'hello world' file_progress_listener = mock.MagicMock() large_file_upload_state = LargeFileUploadState(file_progress_listener) large_file_upload_state.set_error('test error') try: self.bucket.api.services.upload_manager.upload_part( self.bucket.id_, file1.file_id, UploadSourceBytes(content), 1, large_file_upload_state, ).result() self.fail('should have thrown') except AlreadyFailed: pass class TestListUnfinished(TestCaseWithBucket): def test_empty(self): self.assertEqual([], list(self.bucket.list_unfinished_large_files())) def test_one(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) self.assertEqual([file1], list(self.bucket.list_unfinished_large_files())) def test_three(self): file1 = self.bucket.start_large_file('file1.txt', 'text/plain', {}) file2 = self.bucket.start_large_file('file2.txt', 'text/plain', {}) file3 = self.bucket.start_large_file('file3.txt', 'text/plain', {}) self.assertEqual( [file1, file2, file3], list(self.bucket.list_unfinished_large_files(batch_size=1)) ) def _make_file(self, file_id, file_name): return self.bucket.start_large_file(file_name, 'text/plain', {}) class TestGetFileInfo(TestCaseWithBucket): def test_version_by_name(self): data = b'hello world' a_id = self.bucket.upload_bytes(data, 'a').id_ info = self.bucket.get_file_info_by_name('a') self.assertIsInstance(info, FileVersionInfo) expected = ( a_id, 'a', 11, 'upload', 'b2/x-auto', 'none', NO_RETENTION_FILE_SETTING, LegalHold.UNSET, None, ) actual = ( info.id_, info.file_name, info.size, info.action, info.content_type, info.server_side_encryption.mode.value, info.file_retention, info.legal_hold, info.cache_control, ) self.assertEqual(expected, actual) def test_version_by_name_file_lock(self): bucket = self.api.create_bucket( 'my-bucket-with-file-lock', 'allPublic', is_file_lock_enabled=True ) data = b'hello world' legal_hold = LegalHold.ON file_retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 100) bucket.upload_bytes(data, 'a', file_retention=file_retention, legal_hold=legal_hold) file_version = bucket.get_file_info_by_name('a') actual = (file_version.legal_hold, file_version.file_retention) self.assertEqual((legal_hold, file_retention), actual) low_perm_account_info = StubAccountInfo() low_perm_api = B2Api(low_perm_account_info, raw_api=self.simulator) low_perm_key_resp = self.api.create_key( key_name='lowperm', capabilities=[ 'listKeys', 'listBuckets', 'listFiles', 'readFiles', ], ) low_perm_api.authorize_account( 'production', low_perm_key_resp['applicationKeyId'], low_perm_key_resp['applicationKey'] ) low_perm_bucket = low_perm_api.get_bucket_by_name('my-bucket-with-file-lock') file_version = low_perm_bucket.get_file_info_by_name('a') actual = (file_version.legal_hold, file_version.file_retention) expected = (LegalHold.UNKNOWN, FileRetentionSetting(RetentionMode.UNKNOWN)) self.assertEqual(expected, actual) def test_version_by_id(self): data = b'hello world' b_id = self.bucket.upload_bytes(data, 'b').id_ info = self.bucket.get_file_info_by_id(b_id) self.assertIsInstance(info, FileVersionInfo) expected = (b_id, 'b', 11, 'upload', 'b2/x-auto') actual = (info.id_, info.file_name, info.size, info.action, info.content_type) self.assertEqual(expected, actual) class TestLs(TestCaseWithBucket): def test_empty(self): self.assertEqual([], list(self.bucket.ls('foo'))) def test_one_file_at_root(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '') def test_three_files_at_root(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb') self.bucket.upload_bytes(data, 'ccc') expected = [ ('a', 11, 'upload', None), ('bb', 11, 'upload', None), ('ccc', 11, 'upload', None), ] self.assertBucketContents(expected, '') def test_three_files_in_dir(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb/1') self.bucket.upload_bytes(data, 'bb/2/sub1') self.bucket.upload_bytes(data, 'bb/2/sub2') self.bucket.upload_bytes(data, 'bb/3') self.bucket.upload_bytes(data, 'ccc') expected = [ ('bb/1', 11, 'upload', None), ('bb/2/sub1', 11, 'upload', 'bb/2/'), ('bb/3', 11, 'upload', None), ] self.assertBucketContents(expected, 'bb', fetch_count=1) def test_three_files_multiple_versions(self): data = b'hello world' self.bucket.upload_bytes(data, 'a') self.bucket.upload_bytes(data, 'bb/1') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/2') self.bucket.upload_bytes(data, 'bb/3') self.bucket.upload_bytes(data, 'ccc') expected = [ ('9998', 'bb/1', 11, 'upload', None), ('9995', 'bb/2', 11, 'upload', None), ('9996', 'bb/2', 11, 'upload', None), ('9997', 'bb/2', 11, 'upload', None), ('9994', 'bb/3', 11, 'upload', None), ] actual = [ (info.id_, info.file_name, info.size, info.action, folder) for (info, folder) in self.bucket.ls('bb', show_versions=True, fetch_count=1) ] self.assertEqual(expected, actual) def test_started_large_file(self): self.bucket.start_large_file('hello.txt') expected = [('hello.txt', 0, 'start', None)] self.assertBucketContents(expected, '', show_versions=True) def test_hidden_file(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') self.bucket.hide_file('hello.txt') expected = [('hello.txt', 0, 'hide', None), ('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_delete_file_version(self): data = b'hello world' self.bucket.upload_bytes(data, 'hello.txt') files = self.bucket.list_file_names('hello.txt', 1)['files'] file_dict = files[0] file_id = file_dict['fileId'] data = b'hello new world' self.bucket.upload_bytes(data, 'hello.txt') self.bucket.delete_file_version(file_id, 'hello.txt') expected = [('hello.txt', 15, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) class TestListVersions(TestCaseWithBucket): def test_encryption(self): data = b'hello world' a = self.bucket.upload_bytes(data, 'a') a_id = a.id_ self.assertEqual(a.server_side_encryption, SSE_NONE) b = self.bucket.upload_bytes(data, 'b', encryption=SSE_B2_AES) self.assertEqual(b.server_side_encryption, SSE_B2_AES) b_id = b.id_ # c_id = self.bucket.upload_bytes(data, 'c', encryption=SSE_NONE).id_ # TODO self.bucket.copy(a_id, 'd', destination_encryption=SSE_B2_AES) self.bucket.copy( b_id, 'e', destination_encryption=SSE_C_AES, file_info={}, content_type='text/plain' ) actual = [info for info in self.bucket.list_file_versions('a')['files']][0] actual = EncryptionSettingFactory.from_file_version_dict(actual) self.assertEqual(SSE_NONE, actual) # bucket default actual = [info for info in self.bucket.list_file_versions('b')['files']][0] actual = EncryptionSettingFactory.from_file_version_dict(actual) self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 # actual = [info for info in self.bucket.list_file_versions('c')][0] # actual = EncryptionSettingFactory.from_file_version_dict(actual) # self.assertEqual(SSE_NONE, actual) # explicitly requested none actual = [info for info in self.bucket.list_file_versions('d')['files']][0] actual = EncryptionSettingFactory.from_file_version_dict(actual) self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 actual = [info for info in self.bucket.list_file_versions('e')['files']][0] actual = EncryptionSettingFactory.from_file_version_dict(actual) self.assertEqual(SSE_C_AES_NO_SECRET, actual) # explicitly requested sse-c class TestCopyFile(TestCaseWithBucket): def test_copy_without_optional_params(self): file_id = self._make_file() self.bucket.copy_file(file_id, 'hello_new.txt') expected = [('hello.txt', 11, 'upload', None), ('hello_new.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_range(self): file_id = self._make_file() self.bucket.copy_file(file_id, 'hello_new.txt', bytes_range=(3, 9)) expected = [('hello.txt', 11, 'upload', None), ('hello_new.txt', 7, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_invalid_metadata(self): file_id = self._make_file() try: self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.COPY, content_type='application/octet-stream', ) self.fail('should have raised InvalidMetadataDirective') except InvalidMetadataDirective as e: self.assertEqual( 'content_type and file_info should be None when metadata_directive is COPY', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_invalid_metadata_replace(self): file_id = self._make_file() try: self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.REPLACE, ) self.fail('should have raised InvalidMetadataDirective') except InvalidMetadataDirective as e: self.assertEqual( 'content_type cannot be None when metadata_directive is REPLACE', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_replace_metadata(self): file_id = self._make_file() self.bucket.copy_file( file_id, 'hello_new.txt', metadata_directive=MetadataDirectiveMode.REPLACE, content_type='text/plain', ) expected = [ ('hello.txt', 11, 'upload', 'b2/x-auto', None), ('hello_new.txt', 11, 'upload', 'text/plain', None), ] actual = [ (info.file_name, info.size, info.action, info.content_type, folder) for (info, folder) in self.bucket.ls(show_versions=True) ] self.assertEqual(expected, actual) def test_copy_with_unsatisfied_range(self): file_id = self._make_file() try: self.bucket.copy_file( file_id, 'hello_new.txt', bytes_range=(12, 15), ) self.fail('should have raised UnsatisfiableRange') except UnsatisfiableRange as e: self.assertEqual( 'The range in the request is outside the size of the file', str(e), ) expected = [('hello.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_with_different_bucket(self): source_bucket = self.api.create_bucket('source-bucket', 'allPublic') file_id = self._make_file(source_bucket) self.bucket.copy_file(file_id, 'hello_new.txt') def ls(bucket): return [ (info.file_name, info.size, info.action, folder) for (info, folder) in bucket.ls(show_versions=True) ] expected = [('hello.txt', 11, 'upload', None)] self.assertEqual(expected, ls(source_bucket)) expected = [('hello_new.txt', 11, 'upload', None)] self.assertBucketContents(expected, '', show_versions=True) def test_copy_retention(self): for data in [self._make_data(self.simulator.MIN_PART_SIZE * 3), b'hello']: for length in [None, len(data)]: with self.subTest(real_length=len(data), length=length): file_id = self.bucket.upload_bytes(data, 'original_file').id_ resulting_file_version = self.bucket.copy( file_id, 'copied_file', file_retention=FileRetentionSetting(RetentionMode.COMPLIANCE, 100), legal_hold=LegalHold.ON, max_part_size=400, ) self.assertEqual( FileRetentionSetting(RetentionMode.COMPLIANCE, 100), resulting_file_version.file_retention, ) self.assertEqual(LegalHold.ON, resulting_file_version.legal_hold) def test_copy_encryption(self): data = b'hello_world' a = self.bucket.upload_bytes(data, 'a') a_id = a.id_ self.assertEqual(a.server_side_encryption, SSE_NONE) b = self.bucket.upload_bytes(data, 'b', encryption=SSE_B2_AES) self.assertEqual(b.server_side_encryption, SSE_B2_AES) b_id = b.id_ c = self.bucket.upload_bytes(data, 'c', encryption=SSE_C_AES) self.assertEqual(c.server_side_encryption, SSE_C_AES_NO_SECRET) c_id = c.id_ for length in [None, len(data)]: for kwargs, expected_encryption in [ (dict(file_id=a_id, destination_encryption=SSE_B2_AES), SSE_B2_AES), ( dict( file_id=a_id, destination_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain', ), SSE_C_AES_NO_SECRET, ), ( dict( file_id=a_id, destination_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain', ), SSE_C_AES_NO_SECRET, ), (dict(file_id=b_id), SSE_NONE), (dict(file_id=b_id, source_encryption=SSE_B2_AES), SSE_NONE), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_B2_AES, ), SSE_B2_AES, ), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain', ), SSE_C_AES_NO_SECRET, ), ( dict( file_id=b_id, source_encryption=SSE_B2_AES, destination_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain', ), SSE_C_AES_NO_SECRET, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, file_info={'new': 'value'}, content_type='text/plain', ), SSE_NONE, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, source_file_info={'old': 'value'}, source_content_type='text/plain', ), SSE_NONE, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES ), SSE_C_AES_NO_SECRET, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_B2_AES, source_file_info={'old': 'value'}, source_content_type='text/plain', ), SSE_B2_AES, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_B2_AES, file_info={'new': 'value'}, content_type='text/plain', ), SSE_B2_AES, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES_2, source_file_info={'old': 'value'}, source_content_type='text/plain', ), SSE_C_AES_2_NO_SECRET, ), ( dict( file_id=c_id, source_encryption=SSE_C_AES, destination_encryption=SSE_C_AES_2, file_info={'new': 'value'}, content_type='text/plain', ), SSE_C_AES_2_NO_SECRET, ), ]: with self.subTest(kwargs=kwargs, length=length, data=data): file_info = self.bucket.copy(**kwargs, new_file_name='new_file', length=length) self.assertTrue(isinstance(file_info, FileVersionInfo)) self.assertEqual(file_info.server_side_encryption, expected_encryption) def _make_file(self, bucket=None): data = b'hello world' actual_bucket = bucket or self.bucket return actual_bucket.upload_bytes(data, 'hello.txt').id_ class TestUpload(TestCaseWithBucket): def test_upload_bytes(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1') self.assertTrue(isinstance(file_info, FileVersionInfo)) self._check_file_contents('file1', data) def test_upload_bytes_file_retention(self): data = b'hello world' retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 150) file_info = self.bucket.upload_bytes( data, 'file1', file_retention=retention, legal_hold=LegalHold.ON ) self._check_file_contents('file1', data) self.assertEqual(retention, file_info.file_retention) self.assertEqual(LegalHold.ON, file_info.legal_hold) def test_upload_bytes_sse_b2(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1', encryption=SSE_B2_AES) self.assertTrue(isinstance(file_info, FileVersionInfo)) self.assertEqual(file_info.server_side_encryption, SSE_B2_AES) def test_upload_bytes_sse_c(self): data = b'hello world' file_info = self.bucket.upload_bytes(data, 'file1', encryption=SSE_C_AES) self.assertTrue(isinstance(file_info, FileVersionInfo)) self.assertEqual(SSE_C_AES_NO_SECRET, file_info.server_side_encryption) def test_upload_local_file_sse_b2(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1', encryption=SSE_B2_AES) self.assertTrue(isinstance(file_info, FileVersionInfo)) self.assertEqual(file_info.server_side_encryption, SSE_B2_AES) self._check_file_contents('file1', data) def test_upload_local_file_sse_c(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) file_info = self.bucket.upload_local_file(path, 'file1', encryption=SSE_C_AES) self.assertTrue(isinstance(file_info, FileVersionInfo)) self.assertEqual(SSE_C_AES_NO_SECRET, file_info.server_side_encryption) self._check_file_contents('file1', data) def test_upload_local_file_retention(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) retention = FileRetentionSetting(RetentionMode.COMPLIANCE, 150) file_info = self.bucket.upload_local_file( path, 'file1', encryption=SSE_C_AES, file_retention=retention, legal_hold=LegalHold.ON, ) self._check_file_contents('file1', data) self.assertEqual(retention, file_info.file_retention) self.assertEqual(LegalHold.ON, file_info.legal_hold) def test_upload_bytes_progress(self): data = b'hello world' progress_listener = StubProgressListener() self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertTrue(progress_listener.is_valid()) def test_upload_local_file_cache_control(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) cache_control = 'max-age=3600' file_info = self.bucket.upload_local_file(path, 'file1', cache_control=cache_control) self.assertEqual(cache_control, file_info.cache_control) def test_upload_bytes_cache_control(self): data = b'hello world' cache_control = 'max-age=3600' file_info = self.bucket.upload_bytes(data, 'file1', cache_control=cache_control) self.assertEqual(cache_control, file_info.cache_control) def test_upload_local_file(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') data = b'hello world' write_file(path, data) self.bucket.upload_local_file(path, 'file1') self._check_file_contents('file1', data) @pytest.mark.skipif(platform.system() == 'Windows', reason='no os.mkfifo() on Windows') def test_upload_fifo(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') os.mkfifo(path) with self.assertRaises(InvalidUploadSource): self.bucket.upload_local_file(path, 'file1') @pytest.mark.skipif(platform.system() == 'Windows', reason='no os.symlink() on Windows') def test_upload_dead_symlink(self): with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file1') os.symlink('non-existing', path) with self.assertRaises(InvalidUploadSource): self.bucket.upload_local_file(path, 'file1') def test_upload_one_retryable_error(self): self.simulator.set_upload_errors([CanRetry(True)]) data = b'hello world' self.bucket.upload_bytes(data, 'file1') def test_upload_file_one_fatal_error(self): self.simulator.set_upload_errors([CanRetry(False)]) data = b'hello world' with self.assertRaises(CanRetry): self.bucket.upload_bytes(data, 'file1') def test_upload_file_too_many_retryable_errors(self): self.simulator.set_upload_errors([CanRetry(True)] * 6) data = b'hello world' with self.assertRaises(MaxRetriesExceeded): self.bucket.upload_bytes(data, 'file1') def test_upload_large(self): data = self._make_data(self.simulator.MIN_PART_SIZE * 3) progress_listener = StubProgressListener() self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_no_parts(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_all_parts_there(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[:part_size]) self._upload_part(large_file_id, 2, data[part_size : 2 * part_size]) self._upload_part(large_file_id, 3, data[2 * part_size :]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_part_does_not_match(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 3, data[:part_size]) # wrong part number for this data progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_wrong_part_size(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1') self._upload_part(large_file_id, 1, data[: part_size + 1]) # one byte to much progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes(data, 'file1', progress_listener=progress_listener) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_file_info(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1', {'property': 'value1'}) self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes( data, 'file1', progress_listener=progress_listener, file_info={'property': 'value1'} ) self.assertEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def test_upload_large_resume_file_info_does_not_match(self): part_size = self.simulator.MIN_PART_SIZE data = self._make_data(part_size * 3) large_file_id = self._start_large_file('file1', {'property': 'value1'}) self._upload_part(large_file_id, 1, data[:part_size]) progress_listener = StubProgressListener() file_info = self.bucket.upload_bytes( data, 'file1', progress_listener=progress_listener, file_info={'property': 'value2'} ) self.assertNotEqual(large_file_id, file_info.id_) self._check_file_contents('file1', data) self.assertTrue(progress_listener.is_valid()) def _start_large_file(self, file_name, file_info=None): if file_info is None: file_info = {} large_file_info = self.simulator.start_large_file( self.api_url, self.account_auth_token, self.bucket_id, file_name, None, file_info ) return large_file_info['fileId'] def _upload_part(self, large_file_id, part_number, part_data): part_stream = BytesIO(part_data) upload_info = self.simulator.get_upload_part_url( self.api_url, self.account_auth_token, large_file_id ) self.simulator.upload_part( upload_info['uploadUrl'], upload_info['authorizationToken'], part_number, len(part_data), hex_sha1_of_bytes(part_data), part_stream, ) def _check_file_contents(self, file_name, expected_contents): download = DownloadDestBytes() with FileSimulator.dont_check_encryption(): self.bucket.download_file_by_name(file_name, download) self.assertEqual(expected_contents, download.get_bytes_written()) class TestConcatenate(TestCaseWithBucket): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.concatenate(sources, file_name=file_name, encryption=encryption) def test_create_remote(self): data = b'hello world' f1_id = self.bucket.upload_bytes(data, 'f1').id_ f2_id = self.bucket.upload_bytes(data, 'f1').id_ with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file') write_file(path, data) created_file = self._create_remote( [ CopySource(f1_id, length=len(data), offset=0), UploadSourceLocalFile(path), CopySource(f2_id, length=len(data), offset=0), ], file_name='created_file', ) self.assertIsInstance(created_file, FileVersionInfo) actual = ( created_file.id_, created_file.file_name, created_file.size, created_file.server_side_encryption, ) expected = ('9997', 'created_file', 33, SSE_NONE) self.assertEqual(expected, actual) def test_create_remote_encryption(self): for data in [b'hello_world', self._make_data(self.simulator.MIN_PART_SIZE * 3)]: f1_id = self.bucket.upload_bytes(data, 'f1', encryption=SSE_C_AES).id_ f2_id = self.bucket.upload_bytes(data, 'f1', encryption=SSE_C_AES_2).id_ with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file') write_file(path, data) created_file = self._create_remote( [ CopySource(f1_id, length=len(data), offset=0, encryption=SSE_C_AES), UploadSourceLocalFile(path), CopySource(f2_id, length=len(data), offset=0, encryption=SSE_C_AES_2), ], file_name=f'created_file_{len(data)}', encryption=SSE_C_AES, ) self.assertIsInstance(created_file, FileVersionInfo) actual = ( created_file.id_, created_file.file_name, created_file.size, created_file.server_side_encryption, ) expected = ( mock.ANY, f'created_file_{len(data)}', mock.ANY, # FIXME: this should be equal to len(data) * 3, # but there is a problem in the simulator/test code somewhere SSE_C_AES_NO_SECRET, ) self.assertEqual(expected, actual) class TestCreateFile(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.create_file( [wi for wi in WriteIntent.wrap_sources_iterator(sources)], file_name=file_name, encryption=encryption, ) class TestConcatenateStream(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.concatenate_stream(sources, file_name=file_name, encryption=encryption) class TestCreateFileStream(TestConcatenate): def _create_remote(self, sources, file_name, encryption=None): return self.bucket.create_file_stream( [wi for wi in WriteIntent.wrap_sources_iterator(sources)], file_name=file_name, encryption=encryption, ) # Downloads class DownloadTests: DATA = 'abcdefghijklmnopqrs' def setUp(self): super().setUp() self.file_info = self.bucket.upload_bytes(self.DATA.encode(), 'file1') self.encrypted_file_info = self.bucket.upload_bytes( self.DATA.encode(), 'enc_file1', encryption=SSE_C_AES ) self.download_dest = DownloadDestBytes() self.progress_listener = StubProgressListener() def _verify(self, expected_result, check_progress_listener=True): assert self.download_dest.get_bytes_written() == expected_result.encode() if check_progress_listener: valid, reason = self.progress_listener.is_valid_reason( check_progress=False, check_monotonic_progress=True, ) assert valid, reason def test_download_by_id_no_progress(self): self.bucket.download_file_by_id(self.file_info.id_, self.download_dest) self._verify(self.DATA, check_progress_listener=False) def test_download_by_name_no_progress(self): self.bucket.download_file_by_name('file1', self.download_dest) self._verify(self.DATA, check_progress_listener=False) def test_download_by_name_progress(self): self.bucket.download_file_by_name('file1', self.download_dest, self.progress_listener) self._verify(self.DATA) def test_download_by_id_progress(self): self.bucket.download_file_by_id( self.file_info.id_, self.download_dest, self.progress_listener ) self._verify(self.DATA) def test_download_by_id_progress_partial(self): self.bucket.download_file_by_id( self.file_info.id_, self.download_dest, self.progress_listener, range_=(3, 9) ) self._verify('defghij') def test_download_by_id_progress_exact_range(self): self.bucket.download_file_by_id( self.file_info.id_, self.download_dest, self.progress_listener, range_=(0, 18) ) self._verify(self.DATA) def test_download_by_id_progress_range_one_off(self): with self.assertRaises( InvalidRange, msg='A range of 0-19 was requested (size of 20), but cloud could only serve 19 of that', ): self.bucket.download_file_by_id( self.file_info.id_, self.download_dest, self.progress_listener, range_=(0, 19), ) def test_download_by_id_progress_partial_inplace_overwrite(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # ||||||| # vvvvvvv # # 123defghij1234567890 with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file2') download_dest = PreSeekedDownloadDest(seek_target=3, local_file_path=path) data = b'12345678901234567890' write_file(path, data) self.bucket.download_file_by_id( self.file_info.id_, download_dest, self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'123defghij1234567890') def test_download_by_id_progress_partial_shifted_overwrite(self): # LOCAL is # 12345678901234567890 # # and then: # # abcdefghijklmnopqrs # ||||||| # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # \\\\\\\ # ||||||| # vvvvvvv # # 1234567defghij567890 with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'file2') download_dest = PreSeekedDownloadDest(seek_target=7, local_file_path=path) data = b'12345678901234567890' write_file(path, data) self.bucket.download_file_by_id( self.file_info.id_, download_dest, self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'1234567defghij567890') def test_download_by_id_no_progress_encryption(self): self.bucket.download_file_by_id( self.encrypted_file_info.id_, self.download_dest, encryption=SSE_C_AES ) self._verify(self.DATA, check_progress_listener=False) def test_download_by_id_no_progress_wrong_encryption(self): with self.assertRaises(SSECKeyError): self.bucket.download_file_by_id( self.encrypted_file_info.id_, self.download_dest, encryption=SSE_C_AES_2 ) def _check_local_file_contents(self, path, expected_contents): with open(path, 'rb') as f: contents = f.read() self.assertEqual(contents, expected_contents) # download empty file class EmptyFileDownloadScenarioMixin: """use with DownloadTests, but not for TestDownloadParallel as it does not like empty files""" def test_download_by_name_empty_file(self): self.file_info = self.bucket.upload_bytes(b'', 'empty') self.bucket.download_file_by_name('empty', self.download_dest, self.progress_listener) self._verify('') # actual tests class TestDownloadDefault(DownloadTests, EmptyFileDownloadScenarioMixin, TestCaseWithBucket): pass class TestDownloadSimple(DownloadTests, EmptyFileDownloadScenarioMixin, TestCaseWithBucket): def setUp(self): super().setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [SimpleDownloader(force_chunk_size=20)] class TestDownloadParallel(DownloadTests, TestCaseWithBucket): def setUp(self): super().setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [ ParallelDownloader( force_chunk_size=2, max_streams=999, min_part_size=2, ) ] # Truncated downloads class TruncatedFakeResponse(FakeResponse): """ A special FakeResponse class which returns only the first 4 bytes of data. Use it to test followup retries for truncated download issues. """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.data_bytes = self.data_bytes[:4] class TruncatedDownloadBucketSimulator(BucketSimulator): RESPONSE_CLASS = TruncatedFakeResponse class TruncatedDownloadRawSimulator(RawSimulator): BUCKET_SIMULATOR_CLASS = TruncatedDownloadBucketSimulator class TestCaseWithTruncatedDownloadBucket(TestCaseWithBucket): RAW_SIMULATOR_CLASS = TruncatedDownloadRawSimulator ####### actual tests of truncated downloads class TestTruncatedDownloadSimple(DownloadTests, TestCaseWithTruncatedDownloadBucket): def setUp(self): super().setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [SimpleDownloader(force_chunk_size=20)] class TestTruncatedDownloadParallel(DownloadTests, TestCaseWithTruncatedDownloadBucket): def setUp(self): super().setUp() download_manager = self.bucket.api.services.download_manager download_manager.strategies = [ ParallelDownloader( force_chunk_size=3, max_streams=2, min_part_size=2, ) ] b2-sdk-python-2.8.0/test/unit/v0/test_copy_manager.py000066400000000000000000000144151474454370000224770ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_copy_manager.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal.http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME from b2sdk._internal.transfer.outbound.copy_manager import CopyManager from ..test_base import TestBase from .deps import ( SSE_B2_AES, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, MetadataDirectiveMode, ) from .deps_exception import SSECKeyIdMismatchInCopy SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_key', key_id='some-id'), ) SSE_C_AES_2 = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_other_key', key_id='some-id-2'), ) class TestCopyManager(TestBase): def test_establish_sse_c_replace(self): file_info = {'some_key': 'some_value'} content_type = 'text/plain' ( metadata_directive, new_file_info, new_content_type, ) = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.REPLACE, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info=None, source_content_type=None, ) self.assertEqual( ( MetadataDirectiveMode.REPLACE, {'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id'}, content_type, ), (metadata_directive, new_file_info, new_content_type), ) def test_establish_sse_c_copy_no_enc(self): file_info = {} content_type = 'text/plain' ( metadata_directive, new_file_info, new_content_type, ) = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=None, source_server_side_encryption=None, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, {}, content_type), (metadata_directive, new_file_info, new_content_type), ) def test_establish_sse_c_copy_b2(self): file_info = {} content_type = 'text/plain' ( metadata_directive, new_file_info, new_content_type, ) = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_B2_AES, source_server_side_encryption=None, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, {}, content_type), (metadata_directive, new_file_info, new_content_type), ) def test_establish_sse_c_copy_same_key_id(self): file_info = None content_type = 'text/plain' ( metadata_directive, new_file_info, new_content_type, ) = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, None, content_type), (metadata_directive, new_file_info, new_content_type), ) def test_establish_sse_c_copy_sources_given(self): ( metadata_directive, new_file_info, new_content_type, ) = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=None, destination_content_type=None, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info={ 'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id-2', }, source_content_type='text/plain', ) self.assertEqual( ( MetadataDirectiveMode.REPLACE, {'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id'}, 'text/plain', ), (metadata_directive, new_file_info, new_content_type), ) def test_establish_sse_c_copy_sources_unknown(self): for source_file_info, source_content_type in [ (None, None), ({'a': 'b'}, None), (None, 'text/plain'), ]: with self.subTest( source_file_info=source_file_info, source_content_type=source_content_type ): with self.assertRaises( SSECKeyIdMismatchInCopy, 'attempting to copy file using MetadataDirectiveMode.COPY without providing source_file_info ' 'and source_content_type for differing sse_c_key_ids: source="some-id-2", destination="some-id"', ): CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=None, destination_content_type=None, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info=source_file_info, source_content_type=source_content_type, ) b2-sdk-python-2.8.0/test/unit/v0/test_download_dest.py000066400000000000000000000072211474454370000226560ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_download_dest.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import os import tempfile from ..test_base import TestBase from .deps import ( DownloadDestLocalFile, DownloadDestProgressWrapper, PreSeekedDownloadDest, ProgressListenerForTest, ) class TestDownloadDestLocalFile(TestBase): expected_result = 'hello world' def _make_dest(self, temp_dir): file_path = os.path.join(temp_dir, 'test.txt') return DownloadDestLocalFile(file_path), file_path def test_write_and_set_mod_time(self): """ Check that the file gets written and that its mod time gets set. """ mod_time = 1500222333000 with tempfile.TemporaryDirectory() as temp_dir: download_dest, file_path = self._make_dest(temp_dir) with download_dest.make_file_context( 'file_id', 'file_name', 100, 'content_type', 'sha1', {}, mod_time ) as f: f.write(b'hello world') with open(file_path, 'rb') as f: self.assertEqual( self.expected_result.encode(), f.read(), ) self.assertEqual(mod_time, int(os.path.getmtime(file_path) * 1000)) def test_failed_write_deletes_partial_file(self): with tempfile.TemporaryDirectory() as temp_dir: download_dest, file_path = self._make_dest(temp_dir) try: with download_dest.make_file_context( 'file_id', 'file_name', 100, 'content_type', 'sha1', {}, 1500222333000 ) as f: f.write(b'hello world') raise Exception('test error') except Exception as e: self.assertEqual('test error', str(e)) self.assertFalse(os.path.exists(file_path), msg='failed download should be deleted') class TestPreSeekedDownloadDest(TestDownloadDestLocalFile): expected_result = '123hello world567890' def _make_dest(self, temp_dir): file_path = os.path.join(temp_dir, 'test.txt') with open(file_path, 'wb') as f: f.write(b'12345678901234567890') return PreSeekedDownloadDest(local_file_path=file_path, seek_target=3), file_path class TestDownloadDestProgressWrapper(TestBase): def test_write_and_set_mod_time_and_progress(self): """ Check that the file gets written and that its mod time gets set. """ mod_time = 1500222333000 with tempfile.TemporaryDirectory() as temp_dir: file_path = os.path.join(temp_dir, 'test.txt') download_local_file = DownloadDestLocalFile(file_path) progress_listener = ProgressListenerForTest() download_dest = DownloadDestProgressWrapper(download_local_file, progress_listener) with download_dest.make_file_context( 'file_id', 'file_name', 100, 'content_type', 'sha1', {}, mod_time ) as f: f.write(b'hello world\n') with open(file_path, 'rb') as f: self.assertEqual(b'hello world\n', f.read()) self.assertEqual(mod_time, int(os.path.getmtime(file_path) * 1000)) self.assertEqual( [ 'set_total_bytes(100)', 'bytes_completed(12)', 'close()', ], progress_listener.get_calls(), ) b2-sdk-python-2.8.0/test/unit/v0/test_file_metadata.py000066400000000000000000000030521474454370000226050ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_file_metadata.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from ..test_base import TestBase from .deps import FileMetadata def snake_to_camel(name): camel = ''.join(s.title() for s in name.split('_')) return camel[:1].lower() + camel[1:] class TestFileMetadata(TestBase): KWARGS = { 'file_id': '4_deadbeaf3b3e38a957f100d1e_f1042665d79618ae7_d20200903_m194254_c000_v0001053_t0048', 'file_name': 'foo.txt', 'content_type': 'text/plain', 'content_length': '1', 'content_sha1': '4518012e1b365e504001dbc94120624f15b8bbd5', 'file_info': {}, } INFO_DICT = {snake_to_camel(k): v for k, v in KWARGS.items()} def test_verified_sha1(self): metadata = FileMetadata(**self.KWARGS) self.assertTrue(metadata.content_sha1_verified) self.assertEqual(metadata.as_info_dict(), self.INFO_DICT) def test_unverified_sha1(self): kwargs = self.KWARGS.copy() kwargs['content_sha1'] = 'unverified:' + kwargs['content_sha1'] info_dict = self.INFO_DICT.copy() info_dict['contentSha1'] = 'unverified:' + info_dict['contentSha1'] metadata = FileMetadata(**kwargs) self.assertFalse(metadata.content_sha1_verified) self.assertEqual(metadata.as_info_dict(), info_dict) b2-sdk-python-2.8.0/test/unit/v0/test_policy.py000066400000000000000000000064561474454370000213400ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_policy.py # # Copyright 2019, Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from unittest.mock import MagicMock from ..test_base import TestBase from .deps import B2Folder, B2SyncPath, FileVersionInfo, LocalSyncPath, make_b2_keep_days_actions class TestMakeB2KeepDaysActions(TestBase): def setUp(self): self.keep_days = 7 self.today = 100 * 86400 self.one_day_millis = 86400 * 1000 def test_no_versions(self): self.check_one_answer(True, [], []) def test_new_version_no_action(self): self.check_one_answer(True, [(1, -5, 'upload')], []) def test_no_source_one_old_version_hides(self): # An upload that is old gets deleted if there is no source file. self.check_one_answer(False, [(1, -10, 'upload')], ['b2_hide(folder/a)']) def test_old_hide_causes_delete(self): # A hide marker that is old gets deleted, as do the things after it. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'hide'), (3, -20, 'upload')], ['b2_delete(folder/a, 2, (hide marker))', 'b2_delete(folder/a, 3, (old version))'], ) def test_old_upload_causes_delete(self): # An upload that is old stays if there is a source file, but things # behind it go away. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'upload'), (3, -20, 'upload')], ['b2_delete(folder/a, 3, (old version))'], ) def test_out_of_order_dates(self): # The one at date -3 will get deleted because the one before it is old. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'upload'), (3, -3, 'upload')], ['b2_delete(folder/a, 3, (old version))'], ) def check_one_answer(self, has_source, id_relative_date_action_list, expected_actions): source_file = LocalSyncPath('a', 'a', 100, 10) if has_source else None dest_file_versions = [ FileVersionInfo( id_=id_, file_name='folder/' + 'a', upload_timestamp=self.today + relative_date * self.one_day_millis, action=action, size=100, file_info={}, content_type='text/plain', content_sha1='content_sha1', ) for (id_, relative_date, action) in id_relative_date_action_list ] dest_file = ( B2SyncPath('a', selected_version=dest_file_versions[0], all_versions=dest_file_versions) if dest_file_versions else None ) bucket = MagicMock() api = MagicMock() api.get_bucket_by_name.return_value = bucket dest_folder = B2Folder('bucket-1', 'folder', api) actual_actions = list( make_b2_keep_days_actions( source_file, dest_file, dest_folder, dest_folder, self.keep_days, self.today ) ) actual_action_strs = [str(a) for a in actual_actions] self.assertEqual(expected_actions, actual_action_strs) b2-sdk-python-2.8.0/test/unit/v0/test_progress.py000066400000000000000000000040111474454370000216660ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_progress.py # # Copyright 2019, Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from io import BytesIO from ..test_base import TestBase from .deps import StreamWithHash, hex_sha1_of_bytes class TestHashingStream(TestBase): def setUp(self): self.data = b'01234567' self.stream = StreamWithHash(BytesIO(self.data)) self.hash = hex_sha1_of_bytes(self.data) self.expected = self.data + self.hash.encode() def test_no_argument(self): output = self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_less(self): output = self.stream.read(len(self.data) - 1) self.assertEqual(len(output), len(self.data) - 1) output += self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_equal(self): output = self.stream.read(len(self.data)) self.assertEqual(len(output), len(self.data)) output += self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_more(self): output = self.stream.read(len(self.data) + 1) self.assertEqual(len(output), len(self.data) + 1) output += self.stream.read() self.assertEqual(self.expected, output) def test_one_by_one(self): for expected_byte in self.expected: self.assertEqual(bytes((expected_byte,)), self.stream.read(1)) self.assertEqual(b'', self.stream.read(1)) def test_large_read(self): output = self.stream.read(1024) self.assertEqual(self.expected, output) self.assertEqual(b'', self.stream.read(1)) def test_seek_zero(self): output0 = self.stream.read() self.stream.seek(0) output1 = self.stream.read() self.assertEqual(output0, output1) b2-sdk-python-2.8.0/test/unit/v0/test_raw_api.py000066400000000000000000000154561474454370000214630ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_raw_api.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from ..test_base import TestBase from .deps import ( B2Http, B2RawHTTPApi, BucketRetentionSetting, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, RetentionMode, RetentionPeriod, ) from .deps_exception import UnusableFileName, WrongEncryptionModeForBucketDefault # Unicode characters for testing filenames. (0x0394 is a letter Delta.) TWO_BYTE_UNICHR = chr(0x0394) CHAR_UNDER_32 = chr(31) DEL_CHAR = chr(127) class TestRawAPIFilenames(TestBase): """Test that the filename checker passes conforming names and rejects those that don't.""" def setUp(self): self.raw_api = B2RawHTTPApi(B2Http()) def _should_be_ok(self, filename): """Call with test filenames that follow the filename rules. :param filename: unicode (or str) that follows the rules """ print(f'Filename "{filename}" should be OK') self.assertTrue(self.raw_api.check_b2_filename(filename) is None) def _should_raise(self, filename, exception_message): """Call with filenames that don't follow the rules (so the rule checker should raise). :param filename: unicode (or str) that doesn't follow the rules :param exception_message: regexp that matches the exception's detailed message """ print(f'Filename "{filename}" should raise UnusableFileName(".*{exception_message}.*").') with self.assertRaisesRegex(UnusableFileName, exception_message): self.raw_api.check_b2_filename(filename) def test_b2_filename_checker(self): """Test a conforming and non-conforming filename for each rule. From the B2 docs (https://www.backblaze.com/b2/docs/files.html): - Names can be pretty much any UTF-8 string up to 1024 bytes long. - No character codes below 32 are allowed. - Backslashes are not allowed. - DEL characters (127) are not allowed. - File names cannot start with "/", end with "/", or contain "//". - Maximum of 250 bytes of UTF-8 in each segment (part between slashes) of a file name. """ print('test b2 filename rules') # Examples from doc: self._should_be_ok('Kitten Videos') self._should_be_ok('\u81ea\u7531.txt') # Check length # 1024 bytes is ok if the segments are at most 250 chars. s_1024 = 4 * (250 * 'x' + '/') + 20 * 'y' self._should_be_ok(s_1024) # 1025 is too long. self._should_raise(s_1024 + 'x', 'too long') # 1024 bytes with two byte characters should also work. s_1024_two_byte = 4 * (125 * TWO_BYTE_UNICHR + '/') + 20 * 'y' self._should_be_ok(s_1024_two_byte) # But 1025 bytes is too long. self._should_raise(s_1024_two_byte + 'x', 'too long') # Names with unicode values < 32, and DEL aren't allowed. self._should_raise('hey' + CHAR_UNDER_32, 'contains code.*less than 32') # Unicode in the filename shouldn't break the exception message. self._should_raise(TWO_BYTE_UNICHR + CHAR_UNDER_32, 'contains code.*less than 32') self._should_raise(DEL_CHAR, 'DEL.*not allowed') # Names can't start or end with '/' or contain '//' self._should_raise('/hey', 'not start.*/') self._should_raise('hey/', 'not .*end.*/') self._should_raise('not//allowed', 'contain.*//') # Reject segments longer than 250 bytes self._should_raise('foo/' + 251 * 'x', 'segment too long') # So a segment of 125 two-byte chars plus one should also fail. self._should_raise('foo/' + 125 * TWO_BYTE_UNICHR + 'x', 'segment too long') class BucketTestBase: @pytest.fixture(autouse=True) def init(self, mocker): b2_http = mocker.MagicMock() self.raw_api = B2RawHTTPApi(b2_http) class TestUpdateBucket(BucketTestBase): """Test updating bucket.""" @pytest.fixture(autouse=True) def init(self, mocker): b2_http = mocker.MagicMock() self.raw_api = B2RawHTTPApi(b2_http) def test_assertion_raises(self): with pytest.raises(AssertionError): self.raw_api.update_bucket('test', 'account_auth_token', 'account_id', 'bucket_id') @pytest.mark.parametrize( 'bucket_type,bucket_info,default_retention', ( (None, {}, None), ( 'allPublic', None, BucketRetentionSetting(RetentionMode.COMPLIANCE, RetentionPeriod(years=1)), ), ), ) def test_assertion_not_raises(self, bucket_type, bucket_info, default_retention): self.raw_api.update_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', bucket_type=bucket_type, bucket_info=bucket_info, default_retention=default_retention, ) @pytest.mark.parametrize( 'encryption_setting,', ( EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(b'key', 'key-id'), ), EncryptionSetting( mode=EncryptionMode.UNKNOWN, ), ), ) def test_update_bucket_wrong_encryption(self, encryption_setting): with pytest.raises(WrongEncryptionModeForBucketDefault): self.raw_api.update_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', default_server_side_encryption=encryption_setting, bucket_type='allPublic', ) class TestCreateBucket(BucketTestBase): """Test creating bucket.""" @pytest.mark.parametrize( 'encryption_setting,', ( EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(b'key', 'key-id'), ), EncryptionSetting( mode=EncryptionMode.UNKNOWN, ), ), ) def test_create_bucket_wrong_encryption(self, encryption_setting): with pytest.raises(WrongEncryptionModeForBucketDefault): self.raw_api.create_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', bucket_type='allPrivate', bucket_info={}, default_server_side_encryption=encryption_setting, ) b2-sdk-python-2.8.0/test/unit/v0/test_scan_policies.py000066400000000000000000000044161474454370000226460ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_scan_policies.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from ..test_base import TestBase from .deps import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from .deps_exception import InvalidArgument class TestScanPolicies(TestBase): def test_default(self): self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_directory('')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_file('')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_directory('a')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_file('a')) def test_exclude_include(self): policy = ScanPoliciesManager(exclude_file_regexes=('a', 'b'), include_file_regexes=('ab',)) self.assertTrue(policy.should_exclude_file('alfa')) self.assertTrue(policy.should_exclude_file('bravo')) self.assertFalse(policy.should_exclude_file('abend')) self.assertFalse(policy.should_exclude_file('charlie')) def test_exclude_dir(self): policy = ScanPoliciesManager(exclude_dir_regexes=('alfa', 'bravo$')) self.assertTrue(policy.should_exclude_directory('alfa')) self.assertTrue(policy.should_exclude_directory('alfa2')) self.assertTrue(policy.should_exclude_directory('alfa/hello')) self.assertTrue(policy.should_exclude_directory('bravo')) self.assertFalse(policy.should_exclude_directory('bravo2')) self.assertFalse(policy.should_exclude_directory('bravo/hello')) self.assertTrue(policy.should_exclude_file('alfa/foo')) self.assertTrue(policy.should_exclude_file('alfa2/hello/foo')) self.assertTrue(policy.should_exclude_file('alfa/hello/foo.txt')) self.assertTrue(policy.should_exclude_file('bravo/foo')) self.assertFalse(policy.should_exclude_file('bravo2/hello/foo')) self.assertTrue(policy.should_exclude_file('bravo/hello/foo.txt')) def test_include_without_exclude(self): with self.assertRaises(InvalidArgument): ScanPoliciesManager(include_file_regexes=('.*',)) b2-sdk-python-2.8.0/test/unit/v0/test_session.py000066400000000000000000000063761474454370000215250ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_session.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import unittest.mock as mock from ..test_base import TestBase from .deps import ALL_CAPABILITIES, B2Session from .deps_exception import InvalidAuthToken, Unauthorized class TestB2Session(TestBase): def setUp(self): self.account_info = mock.MagicMock() self.account_info.get_account_auth_token.return_value = 'auth_token' self.api = mock.MagicMock() self.api.account_info = self.account_info self.raw_api = mock.MagicMock() self.raw_api.get_file_info_by_id.__name__ = 'get_file_info_by_id' self.raw_api.get_file_info_by_id.side_effect = ['ok'] self.session = B2Session(self.account_info, raw_api=self.raw_api) def test_works_first_time(self): self.assertEqual('ok', self.session.get_file_info_by_id(None)) def test_works_second_time(self): self.raw_api.get_file_info_by_id.side_effect = [ InvalidAuthToken('message', 'code'), 'ok', ] self.assertEqual('ok', self.session.get_file_info_by_id(None)) def test_fails_second_time(self): self.raw_api.get_file_info_by_id.side_effect = [ InvalidAuthToken('message', 'code'), InvalidAuthToken('message', 'code'), ] with self.assertRaises(InvalidAuthToken): self.session.get_file_info_by_id(None) def test_app_key_info_no_info(self): self.account_info.get_allowed.return_value = dict( bucketId=None, bucketName=None, capabilities=ALL_CAPABILITIES, namePrefix=None, ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('no_go', 'code') with self.assertRaisesRegex( Unauthorized, r'no_go for application key with no restrictions \(code\)' ): self.session.get_file_info_by_id(None) def test_app_key_info_no_info_no_message(self): self.account_info.get_allowed.return_value = dict( bucketId=None, bucketName=None, capabilities=ALL_CAPABILITIES, namePrefix=None, ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('', 'code') with self.assertRaisesRegex( Unauthorized, r'unauthorized for application key with no restrictions \(code\)' ): self.session.get_file_info_by_id(None) def test_app_key_info_all_info(self): self.account_info.get_allowed.return_value = dict( bucketId='123456', bucketName='my-bucket', capabilities=['readFiles'], namePrefix='prefix/', ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('no_go', 'code') with self.assertRaisesRegex( Unauthorized, r"no_go for application key with capabilities 'readFiles', restricted to bucket 'my-bucket', restricted to files that start with 'prefix/' \(code\)", ): self.session.get_file_info_by_id(None) b2-sdk-python-2.8.0/test/unit/v0/test_sync.py000066400000000000000000001026411474454370000210060ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_sync.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import concurrent.futures as futures import os import platform import sys import tempfile import threading import time import unittest from unittest.mock import ANY, MagicMock import pytest from ..test_base import TestBase from .deps import ( DEFAULT_SCAN_MANAGER, B2Folder, B2SyncPath, BoundedQueueExecutor, FileVersionInfo, LocalFolder, LocalSyncPath, ScanPoliciesManager, make_folder_sync_actions, parse_sync_folder, zip_folders, ) from .deps_exception import ( CommandError, EmptyDirectory, InvalidArgument, NotADirectory, UnableToCreateDirectory, UnSyncableFilename, ) DAY = 86400000 # milliseconds TODAY = DAY * 100 # an arbitrary reference time for testing def write_file(path, contents): parent = os.path.dirname(path) if not os.path.isdir(parent): os.makedirs(parent) with open(path, 'wb') as f: f.write(contents) class TestSync(TestBase): def setUp(self): self.reporter = MagicMock() class TestFolder(TestSync): __test__ = False NAMES = [ '.dot_file', 'hello.', os.path.join('hello', 'a', '1'), os.path.join('hello', 'a', '2'), os.path.join('hello', 'b'), 'hello0', os.path.join('inner', 'a.bin'), os.path.join('inner', 'a.txt'), os.path.join('inner', 'b.bin'), os.path.join('inner', 'b.txt'), os.path.join('inner', 'more', 'a.bin'), os.path.join('inner', 'more', 'a.txt'), '\u81ea\u7531', ] MOD_TIMES = {'.dot_file': TODAY - DAY, 'hello.': TODAY - DAY} def setUp(self): super().setUp() self.root_dir = '' def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False, ): raise NotImplementedError def all_files(self, policies_manager): return list(self.prepare_folder().all_files(self.reporter, policies_manager)) def assert_filtered_files(self, scan_results, expected_scan_results): self.assertEqual(expected_scan_results, list(f.relative_path for f in scan_results)) self.reporter.local_access_error.assert_not_called() def test_exclusions(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.txt', 'inner/b.txt', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*\\.bin',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_all(self): expected_list = [] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclusions_inclusions(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager( exclude_file_regexes=('.*\\.bin',), include_file_regexes=('.*a\\.bin',), ) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_matches_prefix(self): expected_list = [ '.dot_file', 'hello.', 'hello/b', 'hello0', 'inner/b.bin', 'inner/b.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*a',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory(self): expected_list = [ '.dot_file', 'hello.', 'hello0', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager( exclude_dir_regexes=('hello', 'more', 'hello0'), exclude_file_regexes=('inner',) ) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory2(self): expected_list = [ '.dot_file', 'hello.', 'hello0', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_dir_regexes=('hello$', 'inner')) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory_trailing_slash_does_not_match(self): expected_list = [ '.dot_file', 'hello.', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_dir_regexes=('hello$', 'inner/')) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclusion_with_exact_match(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('hello0',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_before_in_range(self): expected_list = [ 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_modified_before=TODAY - 100) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_before_exact(self): expected_list = [ 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_modified_before=TODAY) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_after_in_range(self): expected_list = ['.dot_file', 'hello.'] polices_manager = ScanPoliciesManager(exclude_modified_after=TODAY - 100) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_after_exact(self): expected_list = ['.dot_file', 'hello.'] polices_manager = ScanPoliciesManager(exclude_modified_after=TODAY - DAY) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) class TestLocalFolder(TestFolder): __test__ = True def setUp(self): super().setUp() self.temp_dir = tempfile.TemporaryDirectory() self.root_dir = self.temp_dir.__enter__() def tearDown(self): self.temp_dir.__exit__(*sys.exc_info()) def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False, ): assert not (broken_symlink and invalid_permissions) if platform.system() == 'Windows': pytest.skip('on Windows there are some environment issues with test directory creation') if prepare_files: for relative_path in self.NAMES: self.prepare_file(relative_path) if broken_symlink: os.symlink( os.path.join(self.root_dir, 'non_existant_file'), os.path.join(self.root_dir, 'bad_symlink'), ) elif invalid_permissions: os.chmod(os.path.join(self.root_dir, self.NAMES[0]), 0) return LocalFolder(self.root_dir) def prepare_file(self, relative_path): path = os.path.join(self.root_dir, relative_path) write_file(path, b'') if relative_path in self.MOD_TIMES: mod_time = int(round(self.MOD_TIMES[relative_path] / 1000)) else: mod_time = int(round(TODAY / 1000)) os.utime(path, (mod_time, mod_time)) def test_slash_sorting(self): # '/' should sort between '.' and '0' folder = self.prepare_folder() self.assertEqual(self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter))) self.reporter.local_access_error.assert_not_called() def test_broken_symlink(self): folder = self.prepare_folder(broken_symlink=True) self.assertEqual(self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter))) self.reporter.local_access_error.assert_called_once_with( os.path.join(self.root_dir, 'bad_symlink') ) def test_invalid_permissions(self): folder = self.prepare_folder(invalid_permissions=True) # tests differ depending on the user running them. "root" will # succeed in os.access(path, os.R_OK) even if the permissions of # the file are 0 as implemented on self._prepare_folder(). # use-case: running test suite inside a docker container if not os.access(os.path.join(self.root_dir, self.NAMES[0]), os.R_OK): self.assertEqual( self.NAMES[1:], list(f.relative_path for f in folder.all_files(self.reporter)) ) self.reporter.local_permission_error.assert_called_once_with( os.path.join(self.root_dir, self.NAMES[0]) ) else: self.assertEqual( self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter)) ) def test_syncable_paths(self): syncable_paths = ( ('test.txt', 'test.txt'), ('./a/test.txt', 'a/test.txt'), ('./a/../test.txt', 'test.txt'), ) folder = self.prepare_folder(prepare_files=False) for syncable_path, norm_syncable_path in syncable_paths: expected = os.path.join(self.root_dir, norm_syncable_path.replace('/', os.path.sep)) self.assertEqual(expected, folder.make_full_path(syncable_path)) def test_unsyncable_paths(self): unsyncable_paths = ('../test.txt', 'a/../../test.txt', '/a/test.txt') folder = self.prepare_folder(prepare_files=False) for unsyncable_path in unsyncable_paths: with self.assertRaises(UnSyncableFilename): folder.make_full_path(unsyncable_path) class TestB2Folder(TestFolder): __test__ = True FILE_VERSION_INFOS = { os.path.join('inner', 'a.txt'): [ ( FileVersionInfo('a2', 'inner/a.txt', 200, 'text/plain', 'sha1', {}, 2000, 'upload'), '', ), ( FileVersionInfo('a1', 'inner/a.txt', 100, 'text/plain', 'sha1', {}, 1000, 'upload'), '', ), ], os.path.join('inner', 'b.txt'): [ ( FileVersionInfo('b2', 'inner/b.txt', 200, 'text/plain', 'sha1', {}, 1999, 'upload'), '', ), ( FileVersionInfo('bs', 'inner/b.txt', 150, 'text/plain', 'sha1', {}, 1500, 'start'), '', ), ( FileVersionInfo( 'b1', 'inner/b.txt', 100, 'text/plain', 'sha1', {'src_last_modified_millis': 1001}, 6666, 'upload', ), '', ), ], } def setUp(self): super().setUp() self.bucket = MagicMock() self.bucket.ls.return_value = [] self.api = MagicMock() self.api.get_bucket_by_name.return_value = self.bucket def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False, ): if prepare_files: for relative_path in self.NAMES: self.prepare_file(relative_path, use_file_versions_info) return B2Folder('bucket-name', self.root_dir, self.api) def prepare_file(self, relative_path, use_file_versions_info=False): if use_file_versions_info and relative_path in self.FILE_VERSION_INFOS: self.bucket.ls.return_value.extend(self.FILE_VERSION_INFOS[relative_path]) return if platform.system() == 'Windows': relative_path = relative_path.replace(os.sep, '/') if relative_path in self.MOD_TIMES: self.bucket.ls.return_value.append( ( FileVersionInfo( relative_path, relative_path, 100, 'text/plain', 'sha1', {}, self.MOD_TIMES[relative_path], 'upload', ), self.root_dir, ) ) else: self.bucket.ls.return_value.append( ( FileVersionInfo( relative_path, relative_path, 100, 'text/plain', 'sha1', {}, TODAY, 'upload' ), self.root_dir, ) ) def test_empty(self): folder = self.prepare_folder(prepare_files=False) self.assertEqual([], list(folder.all_files(self.reporter))) def test_multiple_versions(self): # Test two files, to cover the yield within the loop, and the yield without. folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( [ "B2Path(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", "B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])", ], [ str(f) for f in folder.all_files(self.reporter) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ], ) def test_exclude_modified_multiple_versions(self): polices_manager = ScanPoliciesManager( exclude_modified_before=1001, exclude_modified_after=1999 ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( ["B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ str(f) for f in folder.all_files(self.reporter, policies_manager=polices_manager) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ], ) def test_exclude_modified_all_versions(self): polices_manager = ScanPoliciesManager( exclude_modified_before=1500, exclude_modified_after=1500 ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( [], list(folder.all_files(self.reporter, policies_manager=polices_manager)) ) # Path names not allowed to be sync'd on Windows NOT_SYNCD_ON_WINDOWS = [ 'Z:/windows/system32/drivers/etc/hosts', 'a:/Users/.default/test', r'C:\Windows\system32\drivers\mstsc.sys', ] def test_unsyncable_filenames(self): b2_folder = B2Folder('bucket-name', '', self.api) # Test a list of unsyncable file names filenames_to_test = [ '/', # absolute root path '//', '///', '/..', '/../', '/.../', '/../.', '/../..', '/a.txt', '/folder/a.txt', './folder/a.txt', # current dir relative path 'folder/./a.txt', 'folder/folder/.', 'a//b/', # double-slashes 'a///b', 'a////b', '../test', # start with parent dir '../../test', '../../abc/../test', '../../abc/../test/', '../../abc/../.test', 'a/b/c/../d', # parent dir embedded 'a//..//b../..c/', '..a/b../..c/../d..', 'a/../', 'a/../../../../../', 'a/b/c/..', r'\\', # backslash filenames r'\z', r'..\\', r'..\..', r'\..\\', r'\\..\\..', r'\\', r'\\\\', r'\\\\server\\share\\dir\\file', r'\\server\share\dir\file', r'\\?\C\Drive\temp', r'.\\//', r'..\\..//..\\\\', r'.\\a\\..\\b', r'a\\.\\b', ] if platform.system() == 'Windows': filenames_to_test.extend(self.NOT_SYNCD_ON_WINDOWS) for filename in filenames_to_test: self.bucket.ls.return_value = [ (FileVersionInfo('a1', filename, 1, 'text/plain', 'sha1', {}, 1000, 'upload'), '') ] try: list(b2_folder.all_files(self.reporter)) self.fail("should have thrown UnSyncableFilename for: '%s'" % filename) except UnSyncableFilename as e: self.assertTrue(filename in str(e)) def test_syncable_filenames(self): b2_folder = B2Folder('bucket-name', '', self.api) # Test a list of syncable file names filenames_to_test = [ '', ' ', ' / ', ' ./. ', ' ../.. ', '.. / ..', r'.. \ ..', 'file.txt', '.folder/', '..folder/', '..file', r'file/ and\ folder', 'file..', '..file..', 'folder/a.txt..', '..a/b../c../..d/e../', r'folder\test', r'folder\..f..\..f\..f', r'mix/and\match/', r'a\b\c\d', ] # filenames not permitted on Windows *should* be allowed on Linux if platform.system() != 'Windows': filenames_to_test.extend(self.NOT_SYNCD_ON_WINDOWS) for filename in filenames_to_test: self.bucket.ls.return_value = [ (FileVersionInfo('a1', filename, 1, 'text/plain', 'sha1', {}, 1000, 'upload'), '') ] list(b2_folder.all_files(self.reporter)) class FakeLocalFolder(LocalFolder): def __init__(self, local_sync_paths): super().__init__('folder') self.local_sync_paths = local_sync_paths def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): for single_path in self.local_sync_paths: if single_path.relative_path.endswith('/'): if policies_manager.should_exclude_b2_directory(single_path.relative_path): continue else: if policies_manager.should_exclude_local_path(single_path): continue yield single_path def make_full_path(self, name): return '/dir/' + name class FakeB2Folder(B2Folder): def __init__(self, test_files): self.file_versions = [] for test_file in test_files: self.file_versions.extend(self.file_versions_from_file_tuples(*test_file)) super().__init__('test-bucket', 'folder', MagicMock()) def get_file_versions(self): yield from iter(self.file_versions) @classmethod def file_versions_from_file_tuples(cls, name, mod_times, size=10): """ Makes FileVersion objects. Positive modification times are uploads, and negative modification times are hides. It's a hack, but it works. """ try: mod_times = iter(mod_times) except TypeError: mod_times = [mod_times] return [ FileVersionInfo( id_='id_%s_%d' % (name[0], abs(mod_time)), file_name='folder/' + name, upload_timestamp=abs(mod_time), action='upload' if 0 < mod_time else 'hide', size=size, file_info={'in_b2': 'yes'}, content_type='text/plain', content_sha1='content_sha1', ) for mod_time in mod_times ] @classmethod def sync_path_from_file_tuple(cls, name, mod_times, size=10): file_versions = cls.file_versions_from_file_tuples(name, mod_times, size) return B2SyncPath(name, file_versions[0], file_versions) class TestParseSyncFolder(TestBase): def test_b2_double_slash(self): self._check_one('B2Folder(my-bucket, folder/path)', 'b2://my-bucket/folder/path') def test_b2_no_double_slash(self): self._check_one('B2Folder(my-bucket, folder/path)', 'b2:my-bucket/folder/path') def test_b2_trailing_slash(self): self._check_one('B2Folder(my-bucket, a)', 'b2://my-bucket/a/') def test_b2_no_folder(self): self._check_one('B2Folder(my-bucket, )', 'b2://my-bucket') self._check_one('B2Folder(my-bucket, )', 'b2://my-bucket/') def test_local(self): if platform.system() == 'Windows': drive, _ = os.path.splitdrive(os.getcwd()) expected = f'LocalFolder(\\\\?\\{drive}\\foo)' else: expected = 'LocalFolder(/foo)' self._check_one(expected, '/foo') def test_local_trailing_slash(self): if platform.system() == 'Windows': drive, _ = os.path.splitdrive(os.getcwd()) expected = f'LocalFolder(\\\\?\\{drive}\\foo)' else: expected = 'LocalFolder(/foo)' self._check_one(expected, '/foo/') def _check_one(self, expected, to_parse): api = MagicMock() self.assertEqual(expected, str(parse_sync_folder(str(to_parse), api))) class TestFolderExceptions: """There is an exact copy of this class in unit/v1/test_sync.py - TODO: leave only one when migrating tests to sync-like structure. """ @pytest.mark.parametrize( 'exception,msg', [ pytest.param(NotADirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param(Exception, 'is not a directory', marks=pytest.mark.apiver(to_ver=1)), ], ) def test_ensure_present_not_a_dir(self, exception, msg): with tempfile.TemporaryDirectory() as path: file = os.path.join(path, 'clearly_a_file') with open(file, 'w') as f: f.write(' ') folder = parse_sync_folder(file, MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_present() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(UnableToCreateDirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( Exception, 'unable to create directory', marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_ensure_present_unable_to_create(self, exception, msg): with tempfile.TemporaryDirectory() as path: file = os.path.join(path, 'clearly_a_file') with open(file, 'w') as f: f.write(' ') folder = parse_sync_folder(os.path.join(file, 'nonsense'), MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_present() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(EmptyDirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( CommandError, 'Directory .* is empty. Use --allowEmptySource to sync anyway.', marks=pytest.mark.apiver(to_ver=1), ), ], ) def test_ensure_non_empty(self, exception, msg): with tempfile.TemporaryDirectory() as path: folder = parse_sync_folder(path, MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_non_empty() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(InvalidArgument, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( CommandError, "'//' not allowed in path names", marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_double_slash_not_allowed(self, exception, msg): with pytest.raises(exception, match=msg): parse_sync_folder('b2://a//b', MagicMock()) class TestZipFolders(TestSync): def test_empty(self): folder_a = FakeB2Folder([]) folder_b = FakeB2Folder([]) self.assertEqual([], list(zip_folders(folder_a, folder_b, self.reporter))) def test_one_empty(self): file_a1 = LocalSyncPath('a.txt', 'a.txt', 100, 10) folder_a = FakeLocalFolder([file_a1]) folder_b = FakeB2Folder([]) self.assertEqual([(file_a1, None)], list(zip_folders(folder_a, folder_b, self.reporter))) def test_two(self): file_a1 = ('a.txt', 100, 10) file_a2 = ('b.txt', 100, 10) file_a3 = ('d.txt', 100, 10) file_a4 = ('f.txt', 100, 10) file_b1 = ('b.txt', 200, 10) file_b2 = ('e.txt', 200, 10) folder_a = FakeB2Folder([file_a1, file_a2, file_a3, file_a4]) folder_b = FakeB2Folder([file_b1, file_b2]) self.assertEqual( [ (FakeB2Folder.sync_path_from_file_tuple(*file_a1), None), ( FakeB2Folder.sync_path_from_file_tuple(*file_a2), FakeB2Folder.sync_path_from_file_tuple(*file_b1), ), (FakeB2Folder.sync_path_from_file_tuple(*file_a3), None), (None, FakeB2Folder.sync_path_from_file_tuple(*file_b2)), (FakeB2Folder.sync_path_from_file_tuple(*file_a4), None), ], list(zip_folders(folder_a, folder_b, self.reporter)), ) def test_pass_reporter_to_folder(self): """ Check that the zip_folders() function passes the reporter through to both folders. """ folder_a = MagicMock() folder_b = MagicMock() folder_a.all_files = MagicMock(return_value=iter([])) folder_b.all_files = MagicMock(return_value=iter([])) self.assertEqual([], list(zip_folders(folder_a, folder_b, self.reporter))) folder_a.all_files.assert_called_once_with(self.reporter, ANY) folder_b.all_files.assert_called_once_with(self.reporter) class FakeArgs: """ Can be passed to sync code to simulate command-line options. """ def __init__( self, delete=False, keepDays=None, skipNewer=False, replaceNewer=False, compareVersions=None, compareThreshold=None, excludeRegex=None, excludeDirRegex=None, includeRegex=None, debugLogs=True, dryRun=False, allowEmptySource=False, excludeAllSymlinks=False, ): self.delete = delete self.keepDays = keepDays self.skipNewer = skipNewer self.replaceNewer = replaceNewer self.compareVersions = compareVersions self.compareThreshold = compareThreshold if excludeRegex is None: excludeRegex = [] self.excludeRegex = excludeRegex if includeRegex is None: includeRegex = [] self.includeRegex = includeRegex if excludeDirRegex is None: excludeDirRegex = [] self.excludeDirRegex = excludeDirRegex self.debugLogs = debugLogs self.dryRun = dryRun self.allowEmptySource = allowEmptySource self.excludeAllSymlinks = excludeAllSymlinks def local_file(name, mod_time, size=10): """ Makes a File object for a b2 file, with one FileVersion for each modification time given in mod_times. """ return LocalSyncPath(name, name, mod_time, size) class TestExclusions(TestSync): def _check_folder_sync(self, expected_actions, fakeargs): # only local file_a = ('a.txt', 100) file_b = ('b.txt', 100) file_d = ('d/d.txt', 100) file_e = ('e/e.incl', 100) # both local and remote file_bi = ('b.txt.incl', 100) file_z = ('z.incl', 100) # only remote file_c = ('c.txt', 100) local_folder = FakeLocalFolder( [local_file(*f) for f in (file_a, file_b, file_d, file_e, file_bi, file_z)] ) b2_folder = FakeB2Folder([file_bi, file_c, file_z]) policies_manager = ScanPoliciesManager( exclude_dir_regexes=fakeargs.excludeDirRegex, exclude_file_regexes=fakeargs.excludeRegex, include_file_regexes=fakeargs.includeRegex, exclude_all_symlinks=fakeargs.excludeAllSymlinks, ) actions = list( make_folder_sync_actions( local_folder, b2_folder, fakeargs, TODAY, self.reporter, policies_manager ) ) self.assertEqual(expected_actions, [str(a) for a in actions]) def test_file_exclusions_with_delete(self): expected_actions = [ 'b2_upload(/dir/a.txt, folder/a.txt, 100)', 'b2_delete(folder/b.txt.incl, id_b_100, )', 'b2_delete(folder/c.txt, id_c_100, )', 'b2_upload(/dir/d/d.txt, folder/d/d.txt, 100)', 'b2_upload(/dir/e/e.incl, folder/e/e.incl, 100)', ] self._check_folder_sync(expected_actions, FakeArgs(delete=True, excludeRegex=['b\\.txt'])) def test_file_exclusions_inclusions_with_delete(self): expected_actions = [ 'b2_upload(/dir/a.txt, folder/a.txt, 100)', 'b2_delete(folder/b.txt.incl, id_b_100, )', 'b2_delete(folder/c.txt, id_c_100, )', 'b2_upload(/dir/d/d.txt, folder/d/d.txt, 100)', 'b2_upload(/dir/e/e.incl, folder/e/e.incl, 100)', 'b2_upload(/dir/b.txt.incl, folder/b.txt.incl, 100)', ] fakeargs = FakeArgs(delete=True, excludeRegex=['b\\.txt'], includeRegex=['.*\\.incl']) self._check_folder_sync(expected_actions, fakeargs) class TestBoundedQueueExecutor(TestBase): def test_run_more_than_queue_size(self): """ Makes sure that the executor will run more jobs that the queue size, which ensures that the semaphore gets released, even if an exception is thrown. """ raw_executor = futures.ThreadPoolExecutor(1) bounded_executor = BoundedQueueExecutor(raw_executor, 5) class Counter: """ Counts how many times run() is called. """ def __init__(self): self.counter = 0 def run(self): """ Always increments the counter. Sometimes raises an exception. """ self.counter += 1 if self.counter % 2 == 0: raise Exception('test') counter = Counter() for _ in range(10): bounded_executor.submit(counter.run) bounded_executor.shutdown() self.assertEqual(10, counter.counter) def test_wait_for_running_jobs(self): """ Makes sure that no more than queue_limit workers are running at once, which checks that the semaphore is acquired before submitting an action. """ raw_executor = futures.ThreadPoolExecutor(2) bounded_executor = BoundedQueueExecutor(raw_executor, 1) assert_equal = self.assertEqual class CountAtOnce: """ Counts how many threads are running at once. There should never be more than 1 because that's the limit on the bounded executor. """ def __init__(self): self.running_at_once = 0 self.lock = threading.Lock() def run(self): with self.lock: self.running_at_once += 1 assert_equal(1, self.running_at_once) # While we are sleeping here, no other actions should start # running. If they do, they will increment the counter and # fail the above assertion. time.sleep(0.05) with self.lock: self.running_at_once -= 1 self.counter += 1 if self.counter % 2 == 0: raise Exception('test') count_at_once = CountAtOnce() for _ in range(5): bounded_executor.submit(count_at_once.run) bounded_executor.shutdown() if __name__ == '__main__': unittest.main() b2-sdk-python-2.8.0/test/unit/v0/test_utils.py000066400000000000000000000131431474454370000211700ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_utils.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from ..test_base import TestBase from .deps import ( b2_url_decode, b2_url_encode, choose_part_ranges, format_and_scale_fraction, format_and_scale_number, ) # These are from the B2 Docs (https://www.backblaze.com/b2/docs/string_encoding.html) ENCODING_TEST_CASES = [ {'fullyEncoded': '%20', 'minimallyEncoded': '+', 'string': ' '}, {'fullyEncoded': '%21', 'minimallyEncoded': '!', 'string': '!'}, {'fullyEncoded': '%22', 'minimallyEncoded': '%22', 'string': '"'}, {'fullyEncoded': '%23', 'minimallyEncoded': '%23', 'string': '#'}, {'fullyEncoded': '%24', 'minimallyEncoded': '$', 'string': '$'}, {'fullyEncoded': '%25', 'minimallyEncoded': '%25', 'string': '%'}, {'fullyEncoded': '%26', 'minimallyEncoded': '%26', 'string': '&'}, {'fullyEncoded': '%27', 'minimallyEncoded': "'", 'string': "'"}, {'fullyEncoded': '%28', 'minimallyEncoded': '(', 'string': '('}, {'fullyEncoded': '%29', 'minimallyEncoded': ')', 'string': ')'}, {'fullyEncoded': '%2A', 'minimallyEncoded': '*', 'string': '*'}, {'fullyEncoded': '%2B', 'minimallyEncoded': '%2B', 'string': '+'}, {'fullyEncoded': '%2C', 'minimallyEncoded': '%2C', 'string': ','}, {'fullyEncoded': '%2D', 'minimallyEncoded': '-', 'string': '-'}, {'fullyEncoded': '%2E', 'minimallyEncoded': '.', 'string': '.'}, {'fullyEncoded': '/', 'minimallyEncoded': '/', 'string': '/'}, {'fullyEncoded': '%30', 'minimallyEncoded': '0', 'string': '0'}, {'fullyEncoded': '%39', 'minimallyEncoded': '9', 'string': '9'}, {'fullyEncoded': '%3A', 'minimallyEncoded': ':', 'string': ':'}, {'fullyEncoded': '%3B', 'minimallyEncoded': ';', 'string': ';'}, {'fullyEncoded': '%3C', 'minimallyEncoded': '%3C', 'string': '<'}, {'fullyEncoded': '%3D', 'minimallyEncoded': '=', 'string': '='}, {'fullyEncoded': '%3E', 'minimallyEncoded': '%3E', 'string': '>'}, {'fullyEncoded': '%3F', 'minimallyEncoded': '%3F', 'string': '?'}, {'fullyEncoded': '%40', 'minimallyEncoded': '@', 'string': '@'}, {'fullyEncoded': '%41', 'minimallyEncoded': 'A', 'string': 'A'}, {'fullyEncoded': '%5A', 'minimallyEncoded': 'Z', 'string': 'Z'}, {'fullyEncoded': '%5B', 'minimallyEncoded': '%5B', 'string': '['}, {'fullyEncoded': '%5C', 'minimallyEncoded': '%5C', 'string': '\\'}, {'fullyEncoded': '%5D', 'minimallyEncoded': '%5D', 'string': ']'}, {'fullyEncoded': '%5E', 'minimallyEncoded': '%5E', 'string': '^'}, {'fullyEncoded': '%5F', 'minimallyEncoded': '_', 'string': '_'}, {'fullyEncoded': '%60', 'minimallyEncoded': '%60', 'string': '`'}, {'fullyEncoded': '%61', 'minimallyEncoded': 'a', 'string': 'a'}, {'fullyEncoded': '%7A', 'minimallyEncoded': 'z', 'string': 'z'}, {'fullyEncoded': '%7B', 'minimallyEncoded': '%7B', 'string': '{'}, {'fullyEncoded': '%7C', 'minimallyEncoded': '%7C', 'string': '|'}, {'fullyEncoded': '%7D', 'minimallyEncoded': '%7D', 'string': '}'}, {'fullyEncoded': '%7E', 'minimallyEncoded': '~', 'string': '~'}, {'fullyEncoded': '%7F', 'minimallyEncoded': '%7F', 'string': '\u007f'}, { 'fullyEncoded': '%E8%87%AA%E7%94%B1', 'minimallyEncoded': '%E8%87%AA%E7%94%B1', 'string': '\u81ea\u7531', }, {'fullyEncoded': '%F0%90%90%80', 'minimallyEncoded': '%F0%90%90%80', 'string': '\U00010400'}, ] class TestUrlEncoding(TestBase): def test_it(self): for test_case in ENCODING_TEST_CASES: string = test_case['string'] fully_encoded = test_case['fullyEncoded'] minimally_encoded = test_case['minimallyEncoded'] encoded = b2_url_encode(string) expected_encoded = (minimally_encoded, fully_encoded) if encoded not in expected_encoded: print(f'string: {repr(string)} encoded: {encoded} expected: {expected_encoded}') self.assertTrue(encoded in expected_encoded) self.assertEqual(string, b2_url_decode(fully_encoded)) self.assertEqual(string, b2_url_decode(minimally_encoded)) class TestChooseParts(TestBase): def test_it(self): self._check_one([(0, 100), (100, 100)], 200, 100) self._check_one([(0, 149), (149, 150)], 299, 100) self._check_one([(0, 100), (100, 100), (200, 100)], 300, 100) ten_TB = 10 * 1000 * 1000 * 1000 * 1000 one_GB = 1000 * 1000 * 1000 expected = [(i * one_GB, one_GB) for i in range(10000)] actual = choose_part_ranges(ten_TB, 100 * 1000 * 1000) self.assertEqual(expected, actual) def _check_one(self, expected, content_length, min_part_size): self.assertEqual(expected, choose_part_ranges(content_length, min_part_size)) class TestFormatAndScaleNumber(TestBase): def test_it(self): self._check_one('1 B', 1) self._check_one('999 B', 999) self._check_one('1.00 kB', 1000) self._check_one('999 kB', 999000) def _check_one(self, expected, x): self.assertEqual(expected, format_and_scale_number(x, 'B')) class TestFormatAndScaleFraction(TestBase): def test_it(self): self._check_one('0 / 100 B', 0, 100) self._check_one('0.0 / 10.0 kB', 0, 10000) self._check_one('9.4 / 10.0 kB', 9400, 10000) def _check_one(self, expected, numerator, denominator): self.assertEqual(expected, format_and_scale_fraction(numerator, denominator, 'B')) b2-sdk-python-2.8.0/test/unit/v0/test_version_utils.py000066400000000000000000000076641474454370000227500ustar00rootroot00000000000000###################################################################### # # File: test/unit/v0/test_version_utils.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import warnings from ..test_base import TestBase from .deps import rename_argument, rename_function class TestRenameArgument(TestBase): VERSION = '0.1.10' def test_warning(self): @rename_argument('aaa', 'bbb', '0.1.0', '0.2.0', current_version=self.VERSION) def easy(bbb): """easy docstring""" return bbb # check that warning is not emitted too early with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') assert easy(5) == 5 assert easy(bbb=5) == 5 assert easy.__name__ == 'easy' assert easy.__doc__.strip() == 'easy docstring' assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') assert easy(aaa=5) == 5 assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert ( str(w[-1].message) == "'aaa' is a deprecated argument for 'easy' function/method - it was renamed to 'bbb' in version 0.1.0. Support for the old name is going to be dropped in 0.2.0." ), str(w[-1].message) def test_outdated_replacement(self): with self.assertRaises( AssertionError, msg=f"rename_argument decorator is still used in version {self.VERSION} when old argument name 'aaa' was scheduled to be dropped in 0.1.2. It is time to remove the mapping.", ): @rename_argument('aaa', 'bbb', '0.1.0', '0.1.2', current_version=self.VERSION) def late(bbb): return bbb assert late # make linters happy def test_future_replacement(self): with self.assertRaises( AssertionError, msg="rename_argument decorator indicates that the replacement of argument 'aaa' should take place in the future version 0.2.0, while the current version is 0.2.2. It looks like should be _discouraged_ at this point and not _deprecated_ yet. Consider using 'discourage_argument' decorator instead.", ): @rename_argument('aaa', 'bbb', '0.2.0', '0.2.2', current_version=self.VERSION) def early(bbb): return bbb assert early # make linters happy def test_inverted_versions(self): with self.assertRaises( AssertionError, msg="rename_argument decorator is set to start renaming argument 'aaa' starting at version 0.2.2 and finishing in 0.2.0. It needs to start at a lower version and finish at a higher version.", ): @rename_argument('aaa', 'bbb', '0.2.2', '0.2.0', current_version=self.VERSION) def backwards(bbb): return bbb assert backwards # make linters happy class TestRenameFunction(TestBase): VERSION = '0.1.10' def test_rename_function(self): def new(bbb): return bbb for i in ('new', new): @rename_function(i, '0.1.0', '0.2.0', current_version=self.VERSION) def old(bbb): return bbb with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') assert old(5) == 5 assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert ( str(w[-1].message) == "'old' is deprecated since version 0.1.0 - it was moved to 'new', please switch to use that. The proxy for the old name is going to be removed in 0.2.0." ), str(w[-1].message) b2-sdk-python-2.8.0/test/unit/v1/000077500000000000000000000000001474454370000164165ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/v1/__init__.py000066400000000000000000000005061474454370000205300ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/v1/apiver/000077500000000000000000000000001474454370000177045ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/v1/apiver/__init__.py000066400000000000000000000006651474454370000220240ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/apiver/__init__.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details b2-sdk-python-2.8.0/test/unit/v1/apiver/apiver_deps.py000066400000000000000000000005671474454370000225670ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/apiver/apiver_deps.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk.v1 import * # noqa V = 1 b2-sdk-python-2.8.0/test/unit/v1/apiver/apiver_deps_exception.py000066400000000000000000000006041474454370000246350ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/apiver/apiver_deps_exception.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk.v1.exception import * # noqa b2-sdk-python-2.8.0/test/unit/v1/deps.py000066400000000000000000000011731474454370000177250ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/deps.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # TODO: This module is used in old-style unit tests, written separately for v0 and v1. # It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details from apiver_deps import * b2-sdk-python-2.8.0/test/unit/v1/deps_exception.py000066400000000000000000000012171474454370000220020ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/deps_exception.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # TODO: This module is used in old-style unit tests, written separately for v0 and v1. # It will be removed when all test are rewritten for the new style, like e.g. test/unit/sync/. # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details from apiver_deps_exception import * b2-sdk-python-2.8.0/test/unit/v1/test_bounded_queue_executor.py000066400000000000000000000040201474454370000245650ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_bounded_queue_executor.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import concurrent.futures as futures import time from ..test_base import TestBase from .deps import BoundedQueueExecutor class TestBoundedQueueExecutor(TestBase): def setUp(self): unbounded_executor = futures.ThreadPoolExecutor(max_workers=1) self.executor = BoundedQueueExecutor(unbounded_executor, 1) def tearDown(self): self.executor.shutdown() def test_return_future(self): future_1 = self.executor.submit(lambda: 1) print(future_1) self.assertEqual(1, future_1.result()) def test_blocking(self): # This doesn't actually test that it waits, but it does exercise the code. # Make some futures using a function that takes a little time. def sleep_and_return_fcn(n): def fcn(): time.sleep(0.01) return n return fcn futures = [self.executor.submit(sleep_and_return_fcn(i)) for i in range(10)] # Check the answers answers = list(map(lambda f: f.result(), futures)) self.assertEqual(list(range(10)), answers) def test_no_exceptions(self): f = self.executor.submit(lambda: 1) self.executor.shutdown() self.assertEqual(0, self.executor.get_num_exceptions()) self.assertTrue(f.exception() is None) def test_two_exceptions(self): def thrower(): raise Exception('test_exception') f1 = self.executor.submit(thrower) f2 = self.executor.submit(thrower) self.executor.shutdown() self.assertEqual(2, self.executor.get_num_exceptions()) self.assertFalse(f1.exception() is None) self.assertEqual('test_exception', str(f2.exception())) b2-sdk-python-2.8.0/test/unit/v1/test_copy_manager.py000066400000000000000000000144151474454370000225000ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_copy_manager.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._internal.http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME from b2sdk._internal.transfer.outbound.copy_manager import CopyManager from ..test_base import TestBase from .deps import ( SSE_B2_AES, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, MetadataDirectiveMode, ) from .deps_exception import SSECKeyIdMismatchInCopy SSE_C_AES = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_key', key_id='some-id'), ) SSE_C_AES_2 = EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(secret=b'some_other_key', key_id='some-id-2'), ) class TestCopyManager(TestBase): def test_establish_sse_c_replace(self): file_info = {'some_key': 'some_value'} content_type = 'text/plain' ( metadata_directive, new_file_info, new_content_type, ) = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.REPLACE, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info=None, source_content_type=None, ) self.assertEqual( ( MetadataDirectiveMode.REPLACE, {'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id'}, content_type, ), (metadata_directive, new_file_info, new_content_type), ) def test_establish_sse_c_copy_no_enc(self): file_info = {} content_type = 'text/plain' ( metadata_directive, new_file_info, new_content_type, ) = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=None, source_server_side_encryption=None, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, {}, content_type), (metadata_directive, new_file_info, new_content_type), ) def test_establish_sse_c_copy_b2(self): file_info = {} content_type = 'text/plain' ( metadata_directive, new_file_info, new_content_type, ) = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_B2_AES, source_server_side_encryption=None, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, {}, content_type), (metadata_directive, new_file_info, new_content_type), ) def test_establish_sse_c_copy_same_key_id(self): file_info = None content_type = 'text/plain' ( metadata_directive, new_file_info, new_content_type, ) = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=file_info, destination_content_type=content_type, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES, source_file_info=None, source_content_type=None, ) self.assertEqual( (MetadataDirectiveMode.COPY, None, content_type), (metadata_directive, new_file_info, new_content_type), ) def test_establish_sse_c_copy_sources_given(self): ( metadata_directive, new_file_info, new_content_type, ) = CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=None, destination_content_type=None, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info={ 'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id-2', }, source_content_type='text/plain', ) self.assertEqual( ( MetadataDirectiveMode.REPLACE, {'some_key': 'some_value', SSE_C_KEY_ID_FILE_INFO_KEY_NAME: 'some-id'}, 'text/plain', ), (metadata_directive, new_file_info, new_content_type), ) def test_establish_sse_c_copy_sources_unknown(self): for source_file_info, source_content_type in [ (None, None), ({'a': 'b'}, None), (None, 'text/plain'), ]: with self.subTest( source_file_info=source_file_info, source_content_type=source_content_type ): with self.assertRaises( SSECKeyIdMismatchInCopy, 'attempting to copy file using MetadataDirectiveMode.COPY without providing source_file_info ' 'and source_content_type for differing sse_c_key_ids: source="some-id-2", destination="some-id"', ): CopyManager.establish_sse_c_file_metadata( MetadataDirectiveMode.COPY, destination_file_info=None, destination_content_type=None, destination_server_side_encryption=SSE_C_AES, source_server_side_encryption=SSE_C_AES_2, source_file_info=source_file_info, source_content_type=source_content_type, ) b2-sdk-python-2.8.0/test/unit/v1/test_download_dest.py000066400000000000000000000072211474454370000226570ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_download_dest.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import os import tempfile from ..test_base import TestBase from .deps import ( DownloadDestLocalFile, DownloadDestProgressWrapper, PreSeekedDownloadDest, ProgressListenerForTest, ) class TestDownloadDestLocalFile(TestBase): expected_result = 'hello world' def _make_dest(self, temp_dir): file_path = os.path.join(temp_dir, 'test.txt') return DownloadDestLocalFile(file_path), file_path def test_write_and_set_mod_time(self): """ Check that the file gets written and that its mod time gets set. """ mod_time = 1500222333000 with tempfile.TemporaryDirectory() as temp_dir: download_dest, file_path = self._make_dest(temp_dir) with download_dest.make_file_context( 'file_id', 'file_name', 100, 'content_type', 'sha1', {}, mod_time ) as f: f.write(b'hello world') with open(file_path, 'rb') as f: self.assertEqual( self.expected_result.encode(), f.read(), ) self.assertEqual(mod_time, int(os.path.getmtime(file_path) * 1000)) def test_failed_write_deletes_partial_file(self): with tempfile.TemporaryDirectory() as temp_dir: download_dest, file_path = self._make_dest(temp_dir) try: with download_dest.make_file_context( 'file_id', 'file_name', 100, 'content_type', 'sha1', {}, 1500222333000 ) as f: f.write(b'hello world') raise Exception('test error') except Exception as e: self.assertEqual('test error', str(e)) self.assertFalse(os.path.exists(file_path), msg='failed download should be deleted') class TestPreSeekedDownloadDest(TestDownloadDestLocalFile): expected_result = '123hello world567890' def _make_dest(self, temp_dir): file_path = os.path.join(temp_dir, 'test.txt') with open(file_path, 'wb') as f: f.write(b'12345678901234567890') return PreSeekedDownloadDest(local_file_path=file_path, seek_target=3), file_path class TestDownloadDestProgressWrapper(TestBase): def test_write_and_set_mod_time_and_progress(self): """ Check that the file gets written and that its mod time gets set. """ mod_time = 1500222333000 with tempfile.TemporaryDirectory() as temp_dir: file_path = os.path.join(temp_dir, 'test.txt') download_local_file = DownloadDestLocalFile(file_path) progress_listener = ProgressListenerForTest() download_dest = DownloadDestProgressWrapper(download_local_file, progress_listener) with download_dest.make_file_context( 'file_id', 'file_name', 100, 'content_type', 'sha1', {}, mod_time ) as f: f.write(b'hello world\n') with open(file_path, 'rb') as f: self.assertEqual(b'hello world\n', f.read()) self.assertEqual(mod_time, int(os.path.getmtime(file_path) * 1000)) self.assertEqual( [ 'set_total_bytes(100)', 'bytes_completed(12)', 'close()', ], progress_listener.get_calls(), ) b2-sdk-python-2.8.0/test/unit/v1/test_file_metadata.py000066400000000000000000000030521474454370000226060ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_file_metadata.py # # Copyright 2020 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from ..test_base import TestBase from .deps import FileMetadata def snake_to_camel(name): camel = ''.join(s.title() for s in name.split('_')) return camel[:1].lower() + camel[1:] class TestFileMetadata(TestBase): KWARGS = { 'file_id': '4_deadbeaf3b3e38a957f100d1e_f1042665d79618ae7_d20200903_m194254_c000_v0001053_t0048', 'file_name': 'foo.txt', 'content_type': 'text/plain', 'content_length': '1', 'content_sha1': '4518012e1b365e504001dbc94120624f15b8bbd5', 'file_info': {}, } INFO_DICT = {snake_to_camel(k): v for k, v in KWARGS.items()} def test_verified_sha1(self): metadata = FileMetadata(**self.KWARGS) self.assertTrue(metadata.content_sha1_verified) self.assertEqual(metadata.as_info_dict(), self.INFO_DICT) def test_unverified_sha1(self): kwargs = self.KWARGS.copy() kwargs['content_sha1'] = 'unverified:' + kwargs['content_sha1'] info_dict = self.INFO_DICT.copy() info_dict['contentSha1'] = 'unverified:' + info_dict['contentSha1'] metadata = FileMetadata(**kwargs) self.assertFalse(metadata.content_sha1_verified) self.assertEqual(metadata.as_info_dict(), info_dict) b2-sdk-python-2.8.0/test/unit/v1/test_policy.py000066400000000000000000000064561474454370000213410ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_policy.py # # Copyright 2019, Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from unittest.mock import MagicMock from ..test_base import TestBase from .deps import B2Folder, B2SyncPath, FileVersionInfo, LocalSyncPath, make_b2_keep_days_actions class TestMakeB2KeepDaysActions(TestBase): def setUp(self): self.keep_days = 7 self.today = 100 * 86400 self.one_day_millis = 86400 * 1000 def test_no_versions(self): self.check_one_answer(True, [], []) def test_new_version_no_action(self): self.check_one_answer(True, [(1, -5, 'upload')], []) def test_no_source_one_old_version_hides(self): # An upload that is old gets deleted if there is no source file. self.check_one_answer(False, [(1, -10, 'upload')], ['b2_hide(folder/a)']) def test_old_hide_causes_delete(self): # A hide marker that is old gets deleted, as do the things after it. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'hide'), (3, -20, 'upload')], ['b2_delete(folder/a, 2, (hide marker))', 'b2_delete(folder/a, 3, (old version))'], ) def test_old_upload_causes_delete(self): # An upload that is old stays if there is a source file, but things # behind it go away. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'upload'), (3, -20, 'upload')], ['b2_delete(folder/a, 3, (old version))'], ) def test_out_of_order_dates(self): # The one at date -3 will get deleted because the one before it is old. self.check_one_answer( True, [(1, -5, 'upload'), (2, -10, 'upload'), (3, -3, 'upload')], ['b2_delete(folder/a, 3, (old version))'], ) def check_one_answer(self, has_source, id_relative_date_action_list, expected_actions): source_file = LocalSyncPath('a', 'a', 100, 10) if has_source else None dest_file_versions = [ FileVersionInfo( id_=id_, file_name='folder/' + 'a', upload_timestamp=self.today + relative_date * self.one_day_millis, action=action, size=100, file_info={}, content_type='text/plain', content_sha1='content_sha1', ) for (id_, relative_date, action) in id_relative_date_action_list ] dest_file = ( B2SyncPath('a', selected_version=dest_file_versions[0], all_versions=dest_file_versions) if dest_file_versions else None ) bucket = MagicMock() api = MagicMock() api.get_bucket_by_name.return_value = bucket dest_folder = B2Folder('bucket-1', 'folder', api) actual_actions = list( make_b2_keep_days_actions( source_file, dest_file, dest_folder, dest_folder, self.keep_days, self.today ) ) actual_action_strs = [str(a) for a in actual_actions] self.assertEqual(expected_actions, actual_action_strs) b2-sdk-python-2.8.0/test/unit/v1/test_progress.py000066400000000000000000000040111474454370000216670ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_progress.py # # Copyright 2019, Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from io import BytesIO from ..test_base import TestBase from .deps import StreamWithHash, hex_sha1_of_bytes class TestHashingStream(TestBase): def setUp(self): self.data = b'01234567' self.stream = StreamWithHash(BytesIO(self.data)) self.hash = hex_sha1_of_bytes(self.data) self.expected = self.data + self.hash.encode() def test_no_argument(self): output = self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_less(self): output = self.stream.read(len(self.data) - 1) self.assertEqual(len(output), len(self.data) - 1) output += self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_equal(self): output = self.stream.read(len(self.data)) self.assertEqual(len(output), len(self.data)) output += self.stream.read() self.assertEqual(self.expected, output) def test_no_argument_more(self): output = self.stream.read(len(self.data) + 1) self.assertEqual(len(output), len(self.data) + 1) output += self.stream.read() self.assertEqual(self.expected, output) def test_one_by_one(self): for expected_byte in self.expected: self.assertEqual(bytes((expected_byte,)), self.stream.read(1)) self.assertEqual(b'', self.stream.read(1)) def test_large_read(self): output = self.stream.read(1024) self.assertEqual(self.expected, output) self.assertEqual(b'', self.stream.read(1)) def test_seek_zero(self): output0 = self.stream.read() self.stream.seek(0) output1 = self.stream.read() self.assertEqual(output0, output1) b2-sdk-python-2.8.0/test/unit/v1/test_raw_api.py000066400000000000000000000152351474454370000214570ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_raw_api.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from ..test_base import TestBase from .deps import ( B2Http, B2RawHTTPApi, BucketRetentionSetting, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, RetentionMode, RetentionPeriod, ) from .deps_exception import UnusableFileName, WrongEncryptionModeForBucketDefault # Unicode characters for testing filenames. (0x0394 is a letter Delta.) TWO_BYTE_UNICHR = chr(0x0394) CHAR_UNDER_32 = chr(31) DEL_CHAR = chr(127) class TestRawAPIFilenames(TestBase): """Test that the filename checker passes conforming names and rejects those that don't.""" def setUp(self): self.raw_api = B2RawHTTPApi(B2Http()) def _should_be_ok(self, filename): """Call with test filenames that follow the filename rules. :param filename: unicode (or str) that follows the rules """ print(f'Filename "{filename}" should be OK') self.assertTrue(self.raw_api.check_b2_filename(filename) is None) def _should_raise(self, filename, exception_message): """Call with filenames that don't follow the rules (so the rule checker should raise). :param filename: unicode (or str) that doesn't follow the rules :param exception_message: regexp that matches the exception's detailed message """ print(f'Filename "{filename}" should raise UnusableFileName(".*{exception_message}.*").') with self.assertRaisesRegex(UnusableFileName, exception_message): self.raw_api.check_b2_filename(filename) def test_b2_filename_checker(self): """Test a conforming and non-conforming filename for each rule. From the B2 docs (https://www.backblaze.com/b2/docs/files.html): - Names can be pretty much any UTF-8 string up to 1024 bytes long. - No character codes below 32 are allowed. - Backslashes are not allowed. - DEL characters (127) are not allowed. - File names cannot start with "/", end with "/", or contain "//". - Maximum of 250 bytes of UTF-8 in each segment (part between slashes) of a file name. """ print('test b2 filename rules') # Examples from doc: self._should_be_ok('Kitten Videos') self._should_be_ok('\u81ea\u7531.txt') # Check length # 1024 bytes is ok if the segments are at most 250 chars. s_1024 = 4 * (250 * 'x' + '/') + 20 * 'y' self._should_be_ok(s_1024) # 1025 is too long. self._should_raise(s_1024 + 'x', 'too long') # 1024 bytes with two byte characters should also work. s_1024_two_byte = 4 * (125 * TWO_BYTE_UNICHR + '/') + 20 * 'y' self._should_be_ok(s_1024_two_byte) # But 1025 bytes is too long. self._should_raise(s_1024_two_byte + 'x', 'too long') # Names with unicode values < 32, and DEL aren't allowed. self._should_raise('hey' + CHAR_UNDER_32, 'contains code.*less than 32') # Unicode in the filename shouldn't break the exception message. self._should_raise(TWO_BYTE_UNICHR + CHAR_UNDER_32, 'contains code.*less than 32') self._should_raise(DEL_CHAR, 'DEL.*not allowed') # Names can't start or end with '/' or contain '//' self._should_raise('/hey', 'not start.*/') self._should_raise('hey/', 'not .*end.*/') self._should_raise('not//allowed', 'contain.*//') # Reject segments longer than 250 bytes self._should_raise('foo/' + 251 * 'x', 'segment too long') # So a segment of 125 two-byte chars plus one should also fail. self._should_raise('foo/' + 125 * TWO_BYTE_UNICHR + 'x', 'segment too long') class BucketTestBase: @pytest.fixture(autouse=True) def init(self, mocker): b2_http = mocker.MagicMock() self.raw_api = B2RawHTTPApi(b2_http) class TestUpdateBucket(BucketTestBase): """Test updating bucket.""" def test_assertion_raises(self): with pytest.raises(AssertionError): self.raw_api.update_bucket('test', 'account_auth_token', 'account_id', 'bucket_id') @pytest.mark.parametrize( 'bucket_type,bucket_info,default_retention', ( (None, {}, None), ( 'allPublic', None, BucketRetentionSetting(RetentionMode.COMPLIANCE, RetentionPeriod(years=1)), ), ), ) def test_assertion_not_raises(self, bucket_type, bucket_info, default_retention): self.raw_api.update_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', bucket_type=bucket_type, bucket_info=bucket_info, default_retention=default_retention, ) @pytest.mark.parametrize( 'encryption_setting,', ( EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(b'key', 'key-id'), ), EncryptionSetting( mode=EncryptionMode.UNKNOWN, ), ), ) def test_update_bucket_wrong_encryption(self, encryption_setting): with pytest.raises(WrongEncryptionModeForBucketDefault): self.raw_api.update_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', default_server_side_encryption=encryption_setting, bucket_type='allPublic', ) class TestCreateBucket(BucketTestBase): """Test creating bucket.""" @pytest.mark.parametrize( 'encryption_setting,', ( EncryptionSetting( mode=EncryptionMode.SSE_C, algorithm=EncryptionAlgorithm.AES256, key=EncryptionKey(b'key', 'key-id'), ), EncryptionSetting( mode=EncryptionMode.UNKNOWN, ), ), ) def test_create_bucket_wrong_encryption(self, encryption_setting): with pytest.raises(WrongEncryptionModeForBucketDefault): self.raw_api.create_bucket( 'test', 'account_auth_token', 'account_id', 'bucket_id', bucket_type='allPrivate', bucket_info={}, default_server_side_encryption=encryption_setting, ) b2-sdk-python-2.8.0/test/unit/v1/test_scan_policies.py000066400000000000000000000044161474454370000226470ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_scan_policies.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from ..test_base import TestBase from .deps import DEFAULT_SCAN_MANAGER, ScanPoliciesManager from .deps_exception import InvalidArgument class TestScanPolicies(TestBase): def test_default(self): self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_directory('')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_file('')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_directory('a')) self.assertFalse(DEFAULT_SCAN_MANAGER.should_exclude_file('a')) def test_exclude_include(self): policy = ScanPoliciesManager(exclude_file_regexes=('a', 'b'), include_file_regexes=('ab',)) self.assertTrue(policy.should_exclude_file('alfa')) self.assertTrue(policy.should_exclude_file('bravo')) self.assertFalse(policy.should_exclude_file('abend')) self.assertFalse(policy.should_exclude_file('charlie')) def test_exclude_dir(self): policy = ScanPoliciesManager(exclude_dir_regexes=('alfa', 'bravo$')) self.assertTrue(policy.should_exclude_directory('alfa')) self.assertTrue(policy.should_exclude_directory('alfa2')) self.assertTrue(policy.should_exclude_directory('alfa/hello')) self.assertTrue(policy.should_exclude_directory('bravo')) self.assertFalse(policy.should_exclude_directory('bravo2')) self.assertFalse(policy.should_exclude_directory('bravo/hello')) self.assertTrue(policy.should_exclude_file('alfa/foo')) self.assertTrue(policy.should_exclude_file('alfa2/hello/foo')) self.assertTrue(policy.should_exclude_file('alfa/hello/foo.txt')) self.assertTrue(policy.should_exclude_file('bravo/foo')) self.assertFalse(policy.should_exclude_file('bravo2/hello/foo')) self.assertTrue(policy.should_exclude_file('bravo/hello/foo.txt')) def test_include_without_exclude(self): with self.assertRaises(InvalidArgument): ScanPoliciesManager(include_file_regexes=('.*',)) b2-sdk-python-2.8.0/test/unit/v1/test_session.py000066400000000000000000000063761474454370000215260ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_session.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import unittest.mock as mock from ..test_base import TestBase from .deps import ALL_CAPABILITIES, B2Session from .deps_exception import InvalidAuthToken, Unauthorized class TestB2Session(TestBase): def setUp(self): self.account_info = mock.MagicMock() self.account_info.get_account_auth_token.return_value = 'auth_token' self.api = mock.MagicMock() self.api.account_info = self.account_info self.raw_api = mock.MagicMock() self.raw_api.get_file_info_by_id.__name__ = 'get_file_info_by_id' self.raw_api.get_file_info_by_id.side_effect = ['ok'] self.session = B2Session(self.account_info, raw_api=self.raw_api) def test_works_first_time(self): self.assertEqual('ok', self.session.get_file_info_by_id(None)) def test_works_second_time(self): self.raw_api.get_file_info_by_id.side_effect = [ InvalidAuthToken('message', 'code'), 'ok', ] self.assertEqual('ok', self.session.get_file_info_by_id(None)) def test_fails_second_time(self): self.raw_api.get_file_info_by_id.side_effect = [ InvalidAuthToken('message', 'code'), InvalidAuthToken('message', 'code'), ] with self.assertRaises(InvalidAuthToken): self.session.get_file_info_by_id(None) def test_app_key_info_no_info(self): self.account_info.get_allowed.return_value = dict( bucketId=None, bucketName=None, capabilities=ALL_CAPABILITIES, namePrefix=None, ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('no_go', 'code') with self.assertRaisesRegex( Unauthorized, r'no_go for application key with no restrictions \(code\)' ): self.session.get_file_info_by_id(None) def test_app_key_info_no_info_no_message(self): self.account_info.get_allowed.return_value = dict( bucketId=None, bucketName=None, capabilities=ALL_CAPABILITIES, namePrefix=None, ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('', 'code') with self.assertRaisesRegex( Unauthorized, r'unauthorized for application key with no restrictions \(code\)' ): self.session.get_file_info_by_id(None) def test_app_key_info_all_info(self): self.account_info.get_allowed.return_value = dict( bucketId='123456', bucketName='my-bucket', capabilities=['readFiles'], namePrefix='prefix/', ) self.raw_api.get_file_info_by_id.side_effect = Unauthorized('no_go', 'code') with self.assertRaisesRegex( Unauthorized, r"no_go for application key with capabilities 'readFiles', restricted to bucket 'my-bucket', restricted to files that start with 'prefix/' \(code\)", ): self.session.get_file_info_by_id(None) b2-sdk-python-2.8.0/test/unit/v1/test_sync.py000066400000000000000000001045011474454370000210040ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_sync.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import concurrent.futures as futures import os import platform import sys import tempfile import threading import time import unittest from unittest.mock import ANY, MagicMock import pytest from ..test_base import TestBase from .deps import ( DEFAULT_SCAN_MANAGER, B2Folder, B2SyncPath, BoundedQueueExecutor, CompareVersionMode, FileVersionInfo, KeepOrDeleteMode, LocalFolder, LocalSyncPath, NewerFileSyncMode, ScanPoliciesManager, Synchronizer, parse_sync_folder, zip_folders, ) from .deps_exception import ( CommandError, EmptyDirectory, InvalidArgument, NotADirectory, UnableToCreateDirectory, UnSyncableFilename, ) DAY = 86400000 # milliseconds TODAY = DAY * 100 # an arbitrary reference time for testing def write_file(path, contents): parent = os.path.dirname(path) if not os.path.isdir(parent): os.makedirs(parent) with open(path, 'wb') as f: f.write(contents) class TestSync(TestBase): def setUp(self): self.reporter = MagicMock() class TestFolder(TestSync): __test__ = False NAMES = [ '.dot_file', 'hello.', os.path.join('hello', 'a', '1'), os.path.join('hello', 'a', '2'), os.path.join('hello', 'b'), 'hello0', os.path.join('inner', 'a.bin'), os.path.join('inner', 'a.txt'), os.path.join('inner', 'b.bin'), os.path.join('inner', 'b.txt'), os.path.join('inner', 'more', 'a.bin'), os.path.join('inner', 'more', 'a.txt'), '\u81ea\u7531', ] MOD_TIMES = {'.dot_file': TODAY - DAY, 'hello.': TODAY - DAY} def setUp(self): super().setUp() self.root_dir = '' def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False, ): raise NotImplementedError def all_files(self, policies_manager): return list(self.prepare_folder().all_files(self.reporter, policies_manager)) def assert_filtered_files(self, scan_results, expected_scan_results): self.assertEqual(expected_scan_results, list(f.relative_path for f in scan_results)) self.reporter.local_access_error.assert_not_called() def test_exclusions(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.txt', 'inner/b.txt', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*\\.bin',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_all(self): expected_list = [] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclusions_inclusions(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager( exclude_file_regexes=('.*\\.bin',), include_file_regexes=('.*a\\.bin',), ) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_matches_prefix(self): expected_list = [ '.dot_file', 'hello.', 'hello/b', 'hello0', 'inner/b.bin', 'inner/b.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('.*a',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory(self): expected_list = [ '.dot_file', 'hello.', 'hello0', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager( exclude_dir_regexes=('hello', 'more', 'hello0'), exclude_file_regexes=('inner',) ) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory2(self): expected_list = [ '.dot_file', 'hello.', 'hello0', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_dir_regexes=('hello$', 'inner')) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_directory_trailing_slash_does_not_match(self): expected_list = [ '.dot_file', 'hello.', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_dir_regexes=('hello$', 'inner/')) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclusion_with_exact_match(self): expected_list = [ '.dot_file', 'hello.', 'hello/a/1', 'hello/a/2', 'hello/b', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_file_regexes=('hello0',)) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_before_in_range(self): expected_list = [ 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_modified_before=TODAY - 100) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_before_exact(self): expected_list = [ 'hello/a/1', 'hello/a/2', 'hello/b', 'hello0', 'inner/a.bin', 'inner/a.txt', 'inner/b.bin', 'inner/b.txt', 'inner/more/a.bin', 'inner/more/a.txt', '\u81ea\u7531', ] polices_manager = ScanPoliciesManager(exclude_modified_before=TODAY) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_after_in_range(self): expected_list = ['.dot_file', 'hello.'] polices_manager = ScanPoliciesManager(exclude_modified_after=TODAY - 100) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) def test_exclude_modified_after_exact(self): expected_list = ['.dot_file', 'hello.'] polices_manager = ScanPoliciesManager(exclude_modified_after=TODAY - DAY) files = self.all_files(polices_manager) self.assert_filtered_files(files, expected_list) class TestLocalFolder(TestFolder): __test__ = True def setUp(self): super().setUp() self.temp_dir = tempfile.TemporaryDirectory() self.root_dir = self.temp_dir.__enter__() def tearDown(self): self.temp_dir.__exit__(*sys.exc_info()) def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False, ): assert not (broken_symlink and invalid_permissions) if platform.system() == 'Windows': pytest.skip( 'on Windows there are some environment issues with test directory creation' ) # TODO: fix it if prepare_files: for relative_path in self.NAMES: self.prepare_file(relative_path) if broken_symlink: os.symlink( os.path.join(self.root_dir, 'non_existant_file'), os.path.join(self.root_dir, 'bad_symlink'), ) elif invalid_permissions: os.chmod(os.path.join(self.root_dir, self.NAMES[0]), 0) return LocalFolder(self.root_dir) def prepare_file(self, relative_path): path = os.path.join(self.root_dir, relative_path) write_file(path, b'') if relative_path in self.MOD_TIMES: mod_time = int(round(self.MOD_TIMES[relative_path] / 1000)) else: mod_time = int(round(TODAY / 1000)) os.utime(path, (mod_time, mod_time)) def test_slash_sorting(self): # '/' should sort between '.' and '0' folder = self.prepare_folder() self.assertEqual(self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter))) self.reporter.local_access_error.assert_not_called() def test_broken_symlink(self): folder = self.prepare_folder(broken_symlink=True) self.assertEqual(self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter))) self.reporter.local_access_error.assert_called_once_with( os.path.join(self.root_dir, 'bad_symlink') ) def test_invalid_permissions(self): folder = self.prepare_folder(invalid_permissions=True) # tests differ depending on the user running them. "root" will # succeed in os.access(path, os.R_OK) even if the permissions of # the file are 0 as implemented on self._prepare_folder(). # use-case: running test suite inside a docker container if not os.access(os.path.join(self.root_dir, self.NAMES[0]), os.R_OK): self.assertEqual( self.NAMES[1:], list(f.relative_path for f in folder.all_files(self.reporter)) ) self.reporter.local_permission_error.assert_called_once_with( os.path.join(self.root_dir, self.NAMES[0]) ) else: self.assertEqual( self.NAMES, list(f.relative_path for f in folder.all_files(self.reporter)) ) def test_syncable_paths(self): syncable_paths = ( ('test.txt', 'test.txt'), ('./a/test.txt', 'a/test.txt'), ('./a/../test.txt', 'test.txt'), ) folder = self.prepare_folder(prepare_files=False) for syncable_path, norm_syncable_path in syncable_paths: expected = os.path.join(self.root_dir, norm_syncable_path.replace('/', os.path.sep)) self.assertEqual(expected, folder.make_full_path(syncable_path)) def test_unsyncable_paths(self): unsyncable_paths = ('../test.txt', 'a/../../test.txt', '/a/test.txt') folder = self.prepare_folder(prepare_files=False) for unsyncable_path in unsyncable_paths: with self.assertRaises(UnSyncableFilename): folder.make_full_path(unsyncable_path) class TestB2Folder(TestFolder): __test__ = True FILE_VERSION_INFOS = { os.path.join('inner', 'a.txt'): [ ( FileVersionInfo('a2', 'inner/a.txt', 200, 'text/plain', 'sha1', {}, 2000, 'upload'), '', ), ( FileVersionInfo('a1', 'inner/a.txt', 100, 'text/plain', 'sha1', {}, 1000, 'upload'), '', ), ], os.path.join('inner', 'b.txt'): [ ( FileVersionInfo('b2', 'inner/b.txt', 200, 'text/plain', 'sha1', {}, 1999, 'upload'), '', ), ( FileVersionInfo('bs', 'inner/b.txt', 150, 'text/plain', 'sha1', {}, 1500, 'start'), '', ), ( FileVersionInfo( 'b1', 'inner/b.txt', 100, 'text/plain', 'sha1', {'src_last_modified_millis': 1001}, 6666, 'upload', ), '', ), ], } def setUp(self): super().setUp() self.bucket = MagicMock() self.bucket.ls.return_value = [] self.api = MagicMock() self.api.get_bucket_by_name.return_value = self.bucket def prepare_folder( self, prepare_files=True, broken_symlink=False, invalid_permissions=False, use_file_versions_info=False, ): if prepare_files: for relative_path in self.NAMES: self.prepare_file(relative_path, use_file_versions_info) return B2Folder('bucket-name', self.root_dir, self.api) def prepare_file(self, relative_path, use_file_versions_info=False): if use_file_versions_info and relative_path in self.FILE_VERSION_INFOS: self.bucket.ls.return_value.extend(self.FILE_VERSION_INFOS[relative_path]) return if platform.system() == 'Windows': relative_path = relative_path.replace(os.sep, '/') if relative_path in self.MOD_TIMES: self.bucket.ls.return_value.append( ( FileVersionInfo( relative_path, relative_path, 100, 'text/plain', 'sha1', {}, self.MOD_TIMES[relative_path], 'upload', ), self.root_dir, ) ) else: self.bucket.ls.return_value.append( ( FileVersionInfo( relative_path, relative_path, 100, 'text/plain', 'sha1', {}, TODAY, 'upload' ), self.root_dir, ) ) def test_empty(self): folder = self.prepare_folder(prepare_files=False) self.assertEqual([], list(folder.all_files(self.reporter))) def test_multiple_versions(self): # Test two files, to cover the yield within the loop, and the yield without. folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( [ "B2Path(inner/a.txt, [('a2', 2000, 'upload'), ('a1', 1000, 'upload')])", "B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])", ], [ str(f) for f in folder.all_files(self.reporter) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ], ) def test_exclude_modified_multiple_versions(self): polices_manager = ScanPoliciesManager( exclude_modified_before=1001, exclude_modified_after=1999 ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( ["B2Path(inner/b.txt, [('b2', 1999, 'upload'), ('b1', 1001, 'upload')])"], [ str(f) for f in folder.all_files(self.reporter, policies_manager=polices_manager) if f.relative_path in ('inner/a.txt', 'inner/b.txt') ], ) def test_exclude_modified_all_versions(self): polices_manager = ScanPoliciesManager( exclude_modified_before=1500, exclude_modified_after=1500 ) folder = self.prepare_folder(use_file_versions_info=True) self.assertEqual( [], list(folder.all_files(self.reporter, policies_manager=polices_manager)) ) # Path names not allowed to be sync'd on Windows NOT_SYNCD_ON_WINDOWS = [ 'Z:/windows/system32/drivers/etc/hosts', 'a:/Users/.default/test', r'C:\Windows\system32\drivers\mstsc.sys', ] def test_unsyncable_filenames(self): b2_folder = B2Folder('bucket-name', '', self.api) # Test a list of unsyncable file names filenames_to_test = [ '/', # absolute root path '//', '///', '/..', '/../', '/.../', '/../.', '/../..', '/a.txt', '/folder/a.txt', './folder/a.txt', # current dir relative path 'folder/./a.txt', 'folder/folder/.', 'a//b/', # double-slashes 'a///b', 'a////b', '../test', # start with parent dir '../../test', '../../abc/../test', '../../abc/../test/', '../../abc/../.test', 'a/b/c/../d', # parent dir embedded 'a//..//b../..c/', '..a/b../..c/../d..', 'a/../', 'a/../../../../../', 'a/b/c/..', r'\\', # backslash filenames r'\z', r'..\\', r'..\..', r'\..\\', r'\\..\\..', r'\\', r'\\\\', r'\\\\server\\share\\dir\\file', r'\\server\share\dir\file', r'\\?\C\Drive\temp', r'.\\//', r'..\\..//..\\\\', r'.\\a\\..\\b', r'a\\.\\b', ] if platform.system() == 'Windows': filenames_to_test.extend(self.NOT_SYNCD_ON_WINDOWS) for filename in filenames_to_test: self.bucket.ls.return_value = [ (FileVersionInfo('a1', filename, 1, 'text/plain', 'sha1', {}, 1000, 'upload'), '') ] try: list(b2_folder.all_files(self.reporter)) self.fail("should have thrown UnSyncableFilename for: '%s'" % filename) except UnSyncableFilename as e: self.assertTrue(filename in str(e)) def test_syncable_filenames(self): b2_folder = B2Folder('bucket-name', '', self.api) # Test a list of syncable file names filenames_to_test = [ '', ' ', ' / ', ' ./. ', ' ../.. ', '.. / ..', r'.. \ ..', 'file.txt', '.folder/', '..folder/', '..file', r'file/ and\ folder', 'file..', '..file..', 'folder/a.txt..', '..a/b../c../..d/e../', r'folder\test', r'folder\..f..\..f\..f', r'mix/and\match/', r'a\b\c\d', ] # filenames not permitted on Windows *should* be allowed on Linux if platform.system() != 'Windows': filenames_to_test.extend(self.NOT_SYNCD_ON_WINDOWS) for filename in filenames_to_test: self.bucket.ls.return_value = [ (FileVersionInfo('a1', filename, 1, 'text/plain', 'sha1', {}, 1000, 'upload'), '') ] list(b2_folder.all_files(self.reporter)) class FakeLocalFolder(LocalFolder): def __init__(self, local_sync_paths): super().__init__('folder') self.local_sync_paths = local_sync_paths def all_files(self, reporter, policies_manager=DEFAULT_SCAN_MANAGER): for single_path in self.local_sync_paths: if single_path.relative_path.endswith('/'): if policies_manager.should_exclude_b2_directory(single_path.relative_path): continue else: if policies_manager.should_exclude_local_path(single_path): continue yield single_path def make_full_path(self, name): return '/dir/' + name class FakeB2Folder(B2Folder): def __init__(self, test_files): self.file_versions = [] for test_file in test_files: self.file_versions.extend(self.file_versions_from_file_tuples(*test_file)) super().__init__('test-bucket', 'folder', MagicMock()) def get_file_versions(self): yield from iter(self.file_versions) @classmethod def file_versions_from_file_tuples(cls, name, mod_times, size=10): """ Makes FileVersion objects. Positive modification times are uploads, and negative modification times are hides. It's a hack, but it works. """ try: mod_times = iter(mod_times) except TypeError: mod_times = [mod_times] return [ FileVersionInfo( id_='id_%s_%d' % (name[0], abs(mod_time)), file_name='folder/' + name, upload_timestamp=abs(mod_time), action='upload' if 0 < mod_time else 'hide', size=size, file_info={'in_b2': 'yes'}, content_type='text/plain', content_sha1='content_sha1', ) for mod_time in mod_times ] @classmethod def sync_path_from_file_tuple(cls, name, mod_times, size=10): file_versions = cls.file_versions_from_file_tuples(name, mod_times, size) return B2SyncPath(name, file_versions[0], file_versions) class TestParseSyncFolder(TestBase): def test_b2_double_slash(self): self._check_one('B2Folder(my-bucket, folder/path)', 'b2://my-bucket/folder/path') def test_b2_no_double_slash(self): self._check_one('B2Folder(my-bucket, folder/path)', 'b2:my-bucket/folder/path') def test_b2_trailing_slash(self): self._check_one('B2Folder(my-bucket, a)', 'b2://my-bucket/a/') def test_b2_no_folder(self): self._check_one('B2Folder(my-bucket, )', 'b2://my-bucket') self._check_one('B2Folder(my-bucket, )', 'b2://my-bucket/') def test_local(self): if platform.system() == 'Windows': drive, _ = os.path.splitdrive(os.getcwd()) expected = f'LocalFolder(\\\\?\\{drive}\\foo)' else: expected = 'LocalFolder(/foo)' self._check_one(expected, '/foo') def test_local_trailing_slash(self): if platform.system() == 'Windows': drive, _ = os.path.splitdrive(os.getcwd()) expected = f'LocalFolder(\\\\?\\{drive}\\foo)' else: expected = 'LocalFolder(/foo)' self._check_one(expected, '/foo/') def _check_one(self, expected, to_parse): api = MagicMock() self.assertEqual(expected, str(parse_sync_folder(str(to_parse), api))) class TestFolderExceptions: """There is an exact copy of this class in unit/v0/test_sync.py - TODO: leave only one when migrating tests to sync-like structure. """ @pytest.mark.parametrize( 'exception,msg', [ pytest.param(NotADirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param(Exception, 'is not a directory', marks=pytest.mark.apiver(to_ver=1)), ], ) def test_ensure_present_not_a_dir(self, exception, msg): with tempfile.TemporaryDirectory() as path: file = os.path.join(path, 'clearly_a_file') with open(file, 'w') as f: f.write(' ') folder = parse_sync_folder(file, MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_present() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(UnableToCreateDirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( Exception, 'unable to create directory', marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_ensure_present_unable_to_create(self, exception, msg): with tempfile.TemporaryDirectory() as path: file = os.path.join(path, 'clearly_a_file') with open(file, 'w') as f: f.write(' ') folder = parse_sync_folder(os.path.join(file, 'nonsense'), MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_present() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(EmptyDirectory, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( CommandError, 'Directory .* is empty. Use --allowEmptySource to sync anyway.', marks=pytest.mark.apiver(to_ver=1), ), ], ) def test_ensure_non_empty(self, exception, msg): with tempfile.TemporaryDirectory() as path: folder = parse_sync_folder(path, MagicMock()) with pytest.raises(exception, match=msg): folder.ensure_non_empty() @pytest.mark.parametrize( 'exception,msg', [ pytest.param(InvalidArgument, '.*', marks=pytest.mark.apiver(from_ver=2)), pytest.param( CommandError, "'//' not allowed in path names", marks=pytest.mark.apiver(to_ver=1) ), ], ) def test_double_slash_not_allowed(self, exception, msg): with pytest.raises(exception, match=msg): parse_sync_folder('b2://a//b', MagicMock()) class TestZipFolders(TestSync): def test_empty(self): folder_a = FakeB2Folder([]) folder_b = FakeB2Folder([]) self.assertEqual([], list(zip_folders(folder_a, folder_b, self.reporter))) def test_one_empty(self): file_a1 = LocalSyncPath('a.txt', 'a.txt', 100, 10) folder_a = FakeLocalFolder([file_a1]) folder_b = FakeB2Folder([]) self.assertEqual([(file_a1, None)], list(zip_folders(folder_a, folder_b, self.reporter))) def test_two(self): file_a1 = ('a.txt', [100], 10) file_a2 = ('b.txt', [100], 10) file_a3 = ('d.txt', [100], 10) file_a4 = ('f.txt', [100], 10) file_b1 = ('b.txt', [200], 10) file_b2 = ('e.txt', [200], 10) folder_a = FakeB2Folder([file_a1, file_a2, file_a3, file_a4]) folder_b = FakeB2Folder([file_b1, file_b2]) self.assertEqual( [ (FakeB2Folder.sync_path_from_file_tuple(*file_a1), None), ( FakeB2Folder.sync_path_from_file_tuple(*file_a2), FakeB2Folder.sync_path_from_file_tuple(*file_b1), ), (FakeB2Folder.sync_path_from_file_tuple(*file_a3), None), (None, FakeB2Folder.sync_path_from_file_tuple(*file_b2)), (FakeB2Folder.sync_path_from_file_tuple(*file_a4), None), ], list(zip_folders(folder_a, folder_b, self.reporter)), ) def test_pass_reporter_to_folder(self): """ Check that the zip_folders() function passes the reporter through to both folders. """ folder_a = MagicMock() folder_b = MagicMock() folder_a.all_files = MagicMock(return_value=iter([])) folder_b.all_files = MagicMock(return_value=iter([])) self.assertEqual([], list(zip_folders(folder_a, folder_b, self.reporter))) folder_a.all_files.assert_called_once_with(self.reporter, ANY) folder_b.all_files.assert_called_once_with(self.reporter) class FakeArgs: """ Can be passed to sync code to simulate command-line options. """ def __init__( self, newer_file_mode=NewerFileSyncMode.RAISE_ERROR, keep_days_or_delete=KeepOrDeleteMode.NO_DELETE, compare_version_mode=CompareVersionMode.MODTIME, compare_threshold=None, keep_days=None, excludeRegex=None, excludeDirRegex=None, includeRegex=None, debugLogs=True, dryRun=False, allowEmptySource=False, excludeAllSymlinks=False, ): self.newer_file_mode = newer_file_mode self.keep_days_or_delete = keep_days_or_delete self.keep_days = keep_days self.compare_version_mode = compare_version_mode self.compare_threshold = compare_threshold if excludeRegex is None: excludeRegex = [] self.excludeRegex = excludeRegex if includeRegex is None: includeRegex = [] self.includeRegex = includeRegex if excludeDirRegex is None: excludeDirRegex = [] self.excludeDirRegex = excludeDirRegex self.debugLogs = debugLogs self.dryRun = dryRun self.allowEmptySource = allowEmptySource self.excludeAllSymlinks = excludeAllSymlinks def get_synchronizer(self, policies_manager=DEFAULT_SCAN_MANAGER): return Synchronizer( 1, policies_manager=policies_manager, dry_run=self.dryRun, allow_empty_source=self.allowEmptySource, newer_file_mode=self.newer_file_mode, keep_days_or_delete=self.keep_days_or_delete, keep_days=self.keep_days, compare_version_mode=self.compare_version_mode, compare_threshold=self.compare_threshold, ) def local_file(name, mod_time, size=10): """ Makes a LocalSyncPath object for a b2 file. """ return LocalSyncPath(name, name, mod_time, size) class TestExclusions(TestSync): def _check_folder_sync(self, expected_actions, fakeargs): # only local file_a = ('a.txt', 100) file_b = ('b.txt', 100) file_d = ('d/d.txt', 100) file_e = ('e/e.incl', 100) # both local and remote file_bi = ('b.txt.incl', 100) file_z = ('z.incl', 100) # only remote file_c = ('c.txt', 100) local_folder = FakeLocalFolder( [local_file(*f) for f in (file_a, file_b, file_d, file_e, file_bi, file_z)] ) b2_folder = FakeB2Folder([file_bi, file_c, file_z]) policies_manager = ScanPoliciesManager( exclude_dir_regexes=fakeargs.excludeDirRegex, exclude_file_regexes=fakeargs.excludeRegex, include_file_regexes=fakeargs.includeRegex, exclude_all_symlinks=fakeargs.excludeAllSymlinks, ) synchronizer = fakeargs.get_synchronizer(policies_manager=policies_manager) actions = list( synchronizer.make_folder_sync_actions( local_folder, b2_folder, TODAY, self.reporter, policies_manager ) ) self.assertEqual(expected_actions, [str(a) for a in actions]) def test_file_exclusions_with_delete(self): expected_actions = [ 'b2_upload(/dir/a.txt, folder/a.txt, 100)', 'b2_delete(folder/b.txt.incl, id_b_100, )', 'b2_delete(folder/c.txt, id_c_100, )', 'b2_upload(/dir/d/d.txt, folder/d/d.txt, 100)', 'b2_upload(/dir/e/e.incl, folder/e/e.incl, 100)', ] self._check_folder_sync( expected_actions, FakeArgs( keep_days_or_delete=KeepOrDeleteMode.DELETE, excludeRegex=['b\\.txt'], ), ) def test_file_exclusions_inclusions_with_delete(self): expected_actions = [ 'b2_upload(/dir/a.txt, folder/a.txt, 100)', 'b2_delete(folder/b.txt.incl, id_b_100, )', 'b2_delete(folder/c.txt, id_c_100, )', 'b2_upload(/dir/d/d.txt, folder/d/d.txt, 100)', 'b2_upload(/dir/e/e.incl, folder/e/e.incl, 100)', 'b2_upload(/dir/b.txt.incl, folder/b.txt.incl, 100)', ] fakeargs = FakeArgs( keep_days_or_delete=KeepOrDeleteMode.DELETE, excludeRegex=['b\\.txt'], includeRegex=['.*\\.incl'], ) self._check_folder_sync(expected_actions, fakeargs) class TestBoundedQueueExecutor(TestBase): def test_run_more_than_queue_size(self): """ Makes sure that the executor will run more jobs that the queue size, which ensures that the semaphore gets released, even if an exception is thrown. """ raw_executor = futures.ThreadPoolExecutor(1) bounded_executor = BoundedQueueExecutor(raw_executor, 5) class Counter: """ Counts how many times run() is called. """ def __init__(self): self.counter = 0 def run(self): """ Always increments the counter. Sometimes raises an exception. """ self.counter += 1 if self.counter % 2 == 0: raise Exception('test') counter = Counter() for _ in range(10): bounded_executor.submit(counter.run) bounded_executor.shutdown() self.assertEqual(10, counter.counter) def test_wait_for_running_jobs(self): """ Makes sure that no more than queue_limit workers are running at once, which checks that the semaphore is acquired before submitting an action. """ raw_executor = futures.ThreadPoolExecutor(2) bounded_executor = BoundedQueueExecutor(raw_executor, 1) assert_equal = self.assertEqual class CountAtOnce: """ Counts how many threads are running at once. There should never be more than 1 because that's the limit on the bounded executor. """ def __init__(self): self.running_at_once = 0 self.lock = threading.Lock() def run(self): with self.lock: self.running_at_once += 1 assert_equal(1, self.running_at_once) # While we are sleeping here, no other actions should start # running. If they do, they will increment the counter and # fail the above assertion. time.sleep(0.05) with self.lock: self.running_at_once -= 1 self.counter += 1 if self.counter % 2 == 0: raise Exception('test') count_at_once = CountAtOnce() for _ in range(5): bounded_executor.submit(count_at_once.run) bounded_executor.shutdown() if __name__ == '__main__': unittest.main() b2-sdk-python-2.8.0/test/unit/v1/test_utils.py000066400000000000000000000131431474454370000211710ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_utils.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from ..test_base import TestBase from .deps import ( b2_url_decode, b2_url_encode, choose_part_ranges, format_and_scale_fraction, format_and_scale_number, ) # These are from the B2 Docs (https://www.backblaze.com/b2/docs/string_encoding.html) ENCODING_TEST_CASES = [ {'fullyEncoded': '%20', 'minimallyEncoded': '+', 'string': ' '}, {'fullyEncoded': '%21', 'minimallyEncoded': '!', 'string': '!'}, {'fullyEncoded': '%22', 'minimallyEncoded': '%22', 'string': '"'}, {'fullyEncoded': '%23', 'minimallyEncoded': '%23', 'string': '#'}, {'fullyEncoded': '%24', 'minimallyEncoded': '$', 'string': '$'}, {'fullyEncoded': '%25', 'minimallyEncoded': '%25', 'string': '%'}, {'fullyEncoded': '%26', 'minimallyEncoded': '%26', 'string': '&'}, {'fullyEncoded': '%27', 'minimallyEncoded': "'", 'string': "'"}, {'fullyEncoded': '%28', 'minimallyEncoded': '(', 'string': '('}, {'fullyEncoded': '%29', 'minimallyEncoded': ')', 'string': ')'}, {'fullyEncoded': '%2A', 'minimallyEncoded': '*', 'string': '*'}, {'fullyEncoded': '%2B', 'minimallyEncoded': '%2B', 'string': '+'}, {'fullyEncoded': '%2C', 'minimallyEncoded': '%2C', 'string': ','}, {'fullyEncoded': '%2D', 'minimallyEncoded': '-', 'string': '-'}, {'fullyEncoded': '%2E', 'minimallyEncoded': '.', 'string': '.'}, {'fullyEncoded': '/', 'minimallyEncoded': '/', 'string': '/'}, {'fullyEncoded': '%30', 'minimallyEncoded': '0', 'string': '0'}, {'fullyEncoded': '%39', 'minimallyEncoded': '9', 'string': '9'}, {'fullyEncoded': '%3A', 'minimallyEncoded': ':', 'string': ':'}, {'fullyEncoded': '%3B', 'minimallyEncoded': ';', 'string': ';'}, {'fullyEncoded': '%3C', 'minimallyEncoded': '%3C', 'string': '<'}, {'fullyEncoded': '%3D', 'minimallyEncoded': '=', 'string': '='}, {'fullyEncoded': '%3E', 'minimallyEncoded': '%3E', 'string': '>'}, {'fullyEncoded': '%3F', 'minimallyEncoded': '%3F', 'string': '?'}, {'fullyEncoded': '%40', 'minimallyEncoded': '@', 'string': '@'}, {'fullyEncoded': '%41', 'minimallyEncoded': 'A', 'string': 'A'}, {'fullyEncoded': '%5A', 'minimallyEncoded': 'Z', 'string': 'Z'}, {'fullyEncoded': '%5B', 'minimallyEncoded': '%5B', 'string': '['}, {'fullyEncoded': '%5C', 'minimallyEncoded': '%5C', 'string': '\\'}, {'fullyEncoded': '%5D', 'minimallyEncoded': '%5D', 'string': ']'}, {'fullyEncoded': '%5E', 'minimallyEncoded': '%5E', 'string': '^'}, {'fullyEncoded': '%5F', 'minimallyEncoded': '_', 'string': '_'}, {'fullyEncoded': '%60', 'minimallyEncoded': '%60', 'string': '`'}, {'fullyEncoded': '%61', 'minimallyEncoded': 'a', 'string': 'a'}, {'fullyEncoded': '%7A', 'minimallyEncoded': 'z', 'string': 'z'}, {'fullyEncoded': '%7B', 'minimallyEncoded': '%7B', 'string': '{'}, {'fullyEncoded': '%7C', 'minimallyEncoded': '%7C', 'string': '|'}, {'fullyEncoded': '%7D', 'minimallyEncoded': '%7D', 'string': '}'}, {'fullyEncoded': '%7E', 'minimallyEncoded': '~', 'string': '~'}, {'fullyEncoded': '%7F', 'minimallyEncoded': '%7F', 'string': '\u007f'}, { 'fullyEncoded': '%E8%87%AA%E7%94%B1', 'minimallyEncoded': '%E8%87%AA%E7%94%B1', 'string': '\u81ea\u7531', }, {'fullyEncoded': '%F0%90%90%80', 'minimallyEncoded': '%F0%90%90%80', 'string': '\U00010400'}, ] class TestUrlEncoding(TestBase): def test_it(self): for test_case in ENCODING_TEST_CASES: string = test_case['string'] fully_encoded = test_case['fullyEncoded'] minimally_encoded = test_case['minimallyEncoded'] encoded = b2_url_encode(string) expected_encoded = (minimally_encoded, fully_encoded) if encoded not in expected_encoded: print(f'string: {repr(string)} encoded: {encoded} expected: {expected_encoded}') self.assertTrue(encoded in expected_encoded) self.assertEqual(string, b2_url_decode(fully_encoded)) self.assertEqual(string, b2_url_decode(minimally_encoded)) class TestChooseParts(TestBase): def test_it(self): self._check_one([(0, 100), (100, 100)], 200, 100) self._check_one([(0, 149), (149, 150)], 299, 100) self._check_one([(0, 100), (100, 100), (200, 100)], 300, 100) ten_TB = 10 * 1000 * 1000 * 1000 * 1000 one_GB = 1000 * 1000 * 1000 expected = [(i * one_GB, one_GB) for i in range(10000)] actual = choose_part_ranges(ten_TB, 100 * 1000 * 1000) self.assertEqual(expected, actual) def _check_one(self, expected, content_length, min_part_size): self.assertEqual(expected, choose_part_ranges(content_length, min_part_size)) class TestFormatAndScaleNumber(TestBase): def test_it(self): self._check_one('1 B', 1) self._check_one('999 B', 999) self._check_one('1.00 kB', 1000) self._check_one('999 kB', 999000) def _check_one(self, expected, x): self.assertEqual(expected, format_and_scale_number(x, 'B')) class TestFormatAndScaleFraction(TestBase): def test_it(self): self._check_one('0 / 100 B', 0, 100) self._check_one('0.0 / 10.0 kB', 0, 10000) self._check_one('9.4 / 10.0 kB', 9400, 10000) def _check_one(self, expected, numerator, denominator): self.assertEqual(expected, format_and_scale_fraction(numerator, denominator, 'B')) b2-sdk-python-2.8.0/test/unit/v1/test_version_utils.py000066400000000000000000000076701474454370000227460ustar00rootroot00000000000000###################################################################### # # File: test/unit/v1/test_version_utils.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import warnings from b2sdk.v1 import rename_argument, rename_function from ..test_base import TestBase class TestRenameArgument(TestBase): VERSION = '0.1.10' def test_warning(self): @rename_argument('aaa', 'bbb', '0.1.0', '0.2.0', current_version=self.VERSION) def easy(bbb): """easy docstring""" return bbb # check that warning is not emitted too early with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') assert easy(5) == 5 assert easy(bbb=5) == 5 assert easy.__name__ == 'easy' assert easy.__doc__.strip() == 'easy docstring' assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') assert easy(aaa=5) == 5 assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert ( str(w[-1].message) == "'aaa' is a deprecated argument for 'easy' function/method - it was renamed to 'bbb' in version 0.1.0. Support for the old name is going to be dropped in 0.2.0." ), str(w[-1].message) def test_outdated_replacement(self): with self.assertRaises( AssertionError, msg=f"rename_argument decorator is still used in version {self.VERSION} when old argument name 'aaa' was scheduled to be dropped in 0.1.2. It is time to remove the mapping.", ): @rename_argument('aaa', 'bbb', '0.1.0', '0.1.2', current_version=self.VERSION) def late(bbb): return bbb assert late # make linters happy def test_future_replacement(self): with self.assertRaises( AssertionError, msg="rename_argument decorator indicates that the replacement of argument 'aaa' should take place in the future version 0.2.0, while the current version is 0.2.2. It looks like should be _discouraged_ at this point and not _deprecated_ yet. Consider using 'discourage_argument' decorator instead.", ): @rename_argument('aaa', 'bbb', '0.2.0', '0.2.2', current_version=self.VERSION) def early(bbb): return bbb assert early # make linters happy def test_inverted_versions(self): with self.assertRaises( AssertionError, msg="rename_argument decorator is set to start renaming argument 'aaa' starting at version 0.2.2 and finishing in 0.2.0. It needs to start at a lower version and finish at a higher version.", ): @rename_argument('aaa', 'bbb', '0.2.2', '0.2.0', current_version=self.VERSION) def backwards(bbb): return bbb assert backwards # make linters happy class TestRenameFunction(TestBase): VERSION = '0.1.10' def test_rename_function(self): def new(bbb): return bbb for i in ('new', new): @rename_function(i, '0.1.0', '0.2.0', current_version=self.VERSION) def old(bbb): return bbb with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') assert old(5) == 5 assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert ( str(w[-1].message) == "'old' is deprecated since version 0.1.0 - it was moved to 'new', please switch to use that. The proxy for the old name is going to be removed in 0.2.0." ), str(w[-1].message) b2-sdk-python-2.8.0/test/unit/v2/000077500000000000000000000000001474454370000164175ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/v2/__init__.py000066400000000000000000000005061474454370000205310ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/v2/apiver/000077500000000000000000000000001474454370000177055ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/v2/apiver/__init__.py000066400000000000000000000006651474454370000220250ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/apiver/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details b2-sdk-python-2.8.0/test/unit/v2/apiver/apiver_deps.py000066400000000000000000000005671474454370000225700ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/apiver/apiver_deps.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk.v2 import * # noqa V = 2 b2-sdk-python-2.8.0/test/unit/v2/apiver/apiver_deps_exception.py000066400000000000000000000006041474454370000246360ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/apiver/apiver_deps_exception.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk.v2.exception import * # noqa b2-sdk-python-2.8.0/test/unit/v2/conftest.py000066400000000000000000000005061474454370000206170ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/conftest.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/v2/test_bucket.py000066400000000000000000000032471474454370000213130ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/test_bucket.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from unittest.mock import Mock import pytest from b2sdk import _v3 as v3 from b2sdk.v2 import B2Api, Bucket from test.helpers import patch_bind_params @pytest.fixture def dummy_bucket(): return Bucket(Mock(spec=B2Api), 'bucket_id', 'bucket_name') def test_bucket__upload_file__supports_file_infos(dummy_bucket, file_info): """Test v2.Bucket.upload_file support of deprecated file_infos param""" with patch_bind_params(v3.Bucket, 'upload_local_file') as mock_method, pytest.warns( DeprecationWarning, match=r'deprecated argument' ): dummy_bucket.upload_local_file( 'filename', 'filename', file_infos=file_info, ) assert mock_method.get_bound_call_args()['file_info'] == file_info assert 'file_infos' not in mock_method.call_args[1] def test_bucket__upload_bytes__supports_file_infos(dummy_bucket, file_info): """Test v2.Bucket.upload_bytes support of deprecated file_infos param""" with patch_bind_params(dummy_bucket, 'upload') as mock_method, pytest.warns( DeprecationWarning, match=r'deprecated argument' ): dummy_bucket.upload_bytes( b'data', 'filename', file_infos=file_info, ) assert mock_method.get_bound_call_args()['file_info'] == file_info assert 'file_infos' not in mock_method.call_args[1] b2-sdk-python-2.8.0/test/unit/v2/test_raw_api.py000066400000000000000000000042701474454370000214550ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/test_raw_api.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from unittest.mock import Mock import pytest from b2sdk import _v3 as v3 from b2sdk.v2 import B2Http, B2RawHTTPApi from test.helpers import patch_bind_params @pytest.fixture def dummy_b2_raw_http_api(): return B2RawHTTPApi(Mock(spec=B2Http)) def test_b2_raw_http_api__get_upload_file_headers__supports_file_infos( dummy_b2_raw_http_api, file_info ): """Test v2.B2RawHTTPApi.get_upload_file_headers support of deprecated file_infos param""" with patch_bind_params(v3.B2RawHTTPApi, 'get_upload_file_headers') as mock_method, pytest.warns( DeprecationWarning, match=r'deprecated argument' ): dummy_b2_raw_http_api.get_upload_file_headers( 'upload_auth_token', 'file_name', 123, # content_length 'content_type', 'content_sha1', file_infos=file_info, server_side_encryption=None, file_retention=None, legal_hold=None, custom_upload_timestamp=None, ) assert mock_method.get_bound_call_args()['file_info'] == file_info assert 'file_infos' not in mock_method.call_args[1] def test_b2_raw_http_api__upload_file__supports_file_infos(dummy_b2_raw_http_api, file_info): """Test v2.B2RawHTTPApi.upload_file support of deprecated file_infos param""" with patch_bind_params(v3.B2RawHTTPApi, 'upload_file') as mock_method, pytest.warns( DeprecationWarning, match=r'deprecated argument' ): dummy_b2_raw_http_api.upload_file( 'upload_id', 'upload_auth_token', 'file_name', 123, # content_length 'content_type', 'content_sha1', file_infos=file_info, data_stream='data_stream', ) assert mock_method.get_bound_call_args()['file_info'] == file_info assert 'file_infos' not in mock_method.call_args[1] b2-sdk-python-2.8.0/test/unit/v2/test_session.py000066400000000000000000000023301474454370000215110ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/test_session.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from unittest.mock import Mock import pytest from b2sdk import _v3 as v3 from b2sdk.v2 import B2Session from test.helpers import patch_bind_params @pytest.fixture def dummy_session(): return B2Session() def test_session__upload_file__supports_file_infos(dummy_session, file_info): """Test v2.B2Session.upload_file support of deprecated file_infos param""" with patch_bind_params(v3.B2Session, 'upload_file') as mock_method, pytest.warns( DeprecationWarning, match=r'deprecated argument' ): dummy_session.upload_file( 'filename', 'filename', content_type='text/plain', content_length=0, content_sha1='dummy', data_stream=Mock(), file_infos=file_info, ) assert mock_method.get_bound_call_args()['file_info'] == file_info assert 'file_infos' not in mock_method.call_args[1] b2-sdk-python-2.8.0/test/unit/v2/test_utils.py000066400000000000000000000010601474454370000211650ustar00rootroot00000000000000###################################################################### # # File: test/unit/v2/test_utils.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import os import os.path import pytest from .apiver.apiver_deps import TempDir def test_temp_dir() -> None: with pytest.deprecated_call(): with TempDir() as temp_dir: assert os.path.exists(temp_dir) assert not os.path.exists(temp_dir) b2-sdk-python-2.8.0/test/unit/v3/000077500000000000000000000000001474454370000164205ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/v3/__init__.py000066400000000000000000000005061474454370000205320ustar00rootroot00000000000000###################################################################### # # File: test/unit/v3/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/v3/apiver/000077500000000000000000000000001474454370000177065ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/v3/apiver/__init__.py000066400000000000000000000006651474454370000220260ustar00rootroot00000000000000###################################################################### # # File: test/unit/v3/apiver/__init__.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations # configured by pytest using `--api` option # check test/unit/conftest.py:pytest_configure for details b2-sdk-python-2.8.0/test/unit/v3/apiver/apiver_deps.py000066400000000000000000000005701474454370000225630ustar00rootroot00000000000000###################################################################### # # File: test/unit/v3/apiver/apiver_deps.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._v3 import * # noqa V = 3 b2-sdk-python-2.8.0/test/unit/v3/apiver/apiver_deps_exception.py000066400000000000000000000006051474454370000246400ustar00rootroot00000000000000###################################################################### # # File: test/unit/v3/apiver/apiver_deps_exception.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from b2sdk._v3.exception import * # noqa b2-sdk-python-2.8.0/test/unit/v_all/000077500000000000000000000000001474454370000171655ustar00rootroot00000000000000b2-sdk-python-2.8.0/test/unit/v_all/__init__.py000066400000000000000000000005111474454370000212730ustar00rootroot00000000000000###################################################################### # # File: test/unit/v_all/__init__.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations b2-sdk-python-2.8.0/test/unit/v_all/test_api.py000066400000000000000000000127741474454370000213620ustar00rootroot00000000000000###################################################################### # # File: test/unit/v_all/test_api.py # # Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import pytest from apiver_deps import ( B2Api, B2HttpApiConfig, Bucket, EncryptionMode, EncryptionSetting, InMemoryAccountInfo, InMemoryCache, RawSimulator, ) from apiver_deps_exception import BucketIdNotFound from ..test_base import TestBase class DummyA: def __init__(self, *args, **kwargs): pass class DummyB: def __init__(self, *args, **kwargs): pass class TestServices: @pytest.mark.apiver(from_ver=2) @pytest.mark.parametrize( ('kwargs', '_raw_api_class'), [ [ { 'max_upload_workers': 1, 'max_copy_workers': 2, 'max_download_workers': 3, 'save_to_buffer_size': 4, 'check_download_hash': False, 'max_download_streams_per_file': 5, }, DummyA, ], [ { 'max_upload_workers': 2, 'max_copy_workers': 3, 'max_download_workers': 4, 'save_to_buffer_size': 5, 'check_download_hash': True, 'max_download_streams_per_file': 6, }, DummyB, ], ], ) def test_api_initialization(self, kwargs, _raw_api_class): self.account_info = InMemoryAccountInfo() self.cache = InMemoryCache() api_config = B2HttpApiConfig(_raw_api_class=_raw_api_class) self.api = B2Api(self.account_info, self.cache, api_config=api_config, **kwargs) assert self.api.account_info is self.account_info assert self.api.api_config is api_config assert self.api.cache is self.cache assert self.api.session.account_info is self.account_info assert self.api.session.cache is self.cache assert isinstance(self.api.session.raw_api, _raw_api_class) assert isinstance(self.api.file_version_factory, B2Api.FILE_VERSION_FACTORY_CLASS) assert isinstance( self.api.download_version_factory, B2Api.DOWNLOAD_VERSION_FACTORY_CLASS, ) services = self.api.services assert isinstance(services, B2Api.SERVICES_CLASS) # max copy/upload/download workers could only be verified with mocking download_manager = services.download_manager assert isinstance(download_manager, services.DOWNLOAD_MANAGER_CLASS) assert download_manager.write_buffer_size == kwargs['save_to_buffer_size'] assert download_manager.check_hash == kwargs['check_download_hash'] assert download_manager.strategies[0].max_streams == kwargs['max_download_streams_per_file'] class TestApi(TestBase): def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = InMemoryCache() self.api = B2Api( self.account_info, self.cache, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator) ) self.raw_api = self.api.session.raw_api (self.application_key_id, self.master_key) = self.raw_api.create_account() def _authorize_account(self): self.api.authorize_account( realm='production', application_key_id=self.application_key_id, application_key=self.master_key, ) @pytest.mark.apiver(to_ver=1) def test_get_bucket_by_id_up_to_v1(self): bucket = self.api.get_bucket_by_id("this id doesn't even exist") assert bucket.id_ == "this id doesn't even exist" for att_name, att_value in [ ('name', None), ('type_', None), ('bucket_info', {}), ('cors_rules', []), ('lifecycle_rules', []), ('revision', None), ('bucket_dict', {}), ('options_set', set()), ('default_server_side_encryption', EncryptionSetting(EncryptionMode.UNKNOWN)), ]: with self.subTest(att_name=att_name): assert getattr(bucket, att_name) == att_value, att_name @pytest.mark.apiver(from_ver=2) def test_get_bucket_by_id_v2(self): self._authorize_account() with pytest.raises(BucketIdNotFound): self.api.get_bucket_by_id("this id doesn't even exist") created_bucket = self.api.create_bucket('bucket1', 'allPrivate') read_bucket = self.api.get_bucket_by_id(created_bucket.id_) assert created_bucket.id_ == read_bucket.id_ self.cache.save_bucket(Bucket(api=self.api, name='bucket_name', id_='bucket_id')) read_bucket = self.api.get_bucket_by_id('bucket_id') assert read_bucket.name == 'bucket_name' def test_get_download_url_for_file_name(self): self._authorize_account() download_url = self.api.get_download_url_for_file_name('bucket1', 'some-file.txt') assert download_url == 'http://download.example.com/file/bucket1/some-file.txt' def test_get_download_url_for_fileid(self): self._authorize_account() download_url = self.api.get_download_url_for_fileid('file-id') assert ( download_url == 'http://download.example.com/b2api/v3/b2_download_file_by_id?fileId=file-id' ) b2-sdk-python-2.8.0/test/unit/v_all/test_constants.py000066400000000000000000000020161474454370000226110ustar00rootroot00000000000000###################################################################### # # File: test/unit/v_all/test_constants.py # # Copyright 2023 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import apiver_deps import pytest @pytest.mark.apiver(from_ver=2) def test_public_constants(): assert set(dir(apiver_deps)) >= { 'ALL_CAPABILITIES', 'B2_ACCOUNT_INFO_DEFAULT_FILE', 'B2_ACCOUNT_INFO_ENV_VAR', 'B2_ACCOUNT_INFO_PROFILE_FILE', 'DEFAULT_MIN_PART_SIZE', 'DEFAULT_RECOMMENDED_UPLOAD_PART_SIZE', 'LARGE_FILE_SHA1', 'LIST_FILE_NAMES_MAX_LIMIT', 'SERVER_DEFAULT_SYNC_ENCRYPTION_SETTINGS_PROVIDER', 'SRC_LAST_MODIFIED_MILLIS', 'SSE_B2_AES', 'SSE_C_KEY_ID_FILE_INFO_KEY_NAME', 'SSE_NONE', 'UNKNOWN_KEY_ID', 'V', 'VERSION', 'XDG_CONFIG_HOME_ENV_VAR', } b2-sdk-python-2.8.0/test/unit/v_all/test_replication.py000066400000000000000000000115661474454370000231200ustar00rootroot00000000000000###################################################################### # # File: test/unit/v_all/test_replication.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations import logging import pytest from apiver_deps import ( B2Api, B2HttpApiConfig, InMemoryAccountInfo, InMemoryCache, RawSimulator, ReplicationConfiguration, ReplicationRule, ReplicationSetupHelper, ) from ..test_base import TestBase logger = logging.getLogger(__name__) class TestReplication(TestBase): def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = InMemoryCache() self.api = B2Api( self.account_info, self.cache, api_config=B2HttpApiConfig(_raw_api_class=RawSimulator) ) self.raw_api = self.api.session.raw_api self.application_key_id, self.master_key = self.raw_api.create_account() def _authorize_account(self): self.api.authorize_account( realm='production', application_key_id=self.application_key_id, application_key=self.master_key, ) @pytest.mark.apiver(from_ver=2) def test_setup_both(self): self._authorize_account() source_bucket = self.api.create_bucket('bucket1', 'allPrivate') destination_bucket = self.api.create_bucket('bucket2', 'allPrivate') logger.info('preparations complete, starting the test') rsh = ReplicationSetupHelper() source_bucket, destination_bucket = rsh.setup_both( source_bucket=source_bucket, destination_bucket=destination_bucket, name='aa', prefix='ab', ) from pprint import pprint pprint([k.as_dict() for k in self.api.list_keys()]) keymap = {k.key_name: k for k in self.api.list_keys()} source_application_key = keymap['bucket1-replisrc'] assert source_application_key assert set(source_application_key.capabilities) == set( ('readFiles', 'readFileLegalHolds', 'readFileRetentions') ) assert not source_application_key.name_prefix assert source_application_key.expiration_timestamp_millis is None destination_application_key = keymap['bucket2-replidst'] assert destination_application_key assert set(destination_application_key.capabilities) == set( ('writeFiles', 'writeFileLegalHolds', 'writeFileRetentions', 'deleteFiles') ) assert not destination_application_key.name_prefix assert destination_application_key.expiration_timestamp_millis is None assert source_bucket.replication.rules == [ ReplicationRule( destination_bucket_id='bucket_1', name='aa', file_name_prefix='ab', is_enabled=True, priority=128, ) ] assert source_bucket.replication.source_key_id == source_application_key.id_ assert source_bucket.replication.source_to_destination_key_mapping == {} print(destination_bucket.replication) assert destination_bucket.replication.rules == [] assert destination_bucket.replication.source_key_id is None assert destination_bucket.replication.source_to_destination_key_mapping == { source_application_key.id_: destination_application_key.id_ } old_source_application_key = source_application_key source_bucket, destination_bucket = rsh.setup_both( source_bucket=source_bucket, destination_bucket=destination_bucket, prefix='ad', include_existing_files=True, ) keymap = {k.key_name: k for k in self.api.list_keys()} new_source_application_key = keymap['bucket1-replisrc'] assert source_bucket.replication.rules == [ ReplicationRule( destination_bucket_id='bucket_1', name='aa', file_name_prefix='ab', is_enabled=True, priority=128, ), ReplicationRule( destination_bucket_id='bucket_1', name='bucket2', file_name_prefix='ad', is_enabled=True, priority=133, include_existing_files=True, ), ] assert source_bucket.replication.source_key_id == old_source_application_key.id_ assert destination_bucket.replication.source_to_destination_key_mapping == { new_source_application_key.id_: destination_application_key.id_ } @pytest.mark.apiver(from_ver=2) def test_factory(self): replication = ReplicationConfiguration.from_dict({}) assert replication == ReplicationConfiguration() b2-sdk-python-2.8.0/test/unit/v_all/test_transfer.py000066400000000000000000000022041474454370000224200ustar00rootroot00000000000000###################################################################### # # File: test/unit/v_all/test_transfer.py # # Copyright 2022 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### from __future__ import annotations from unittest.mock import Mock from apiver_deps import DownloadManager, UploadManager from ..test_base import TestBase class TestDownloadManager(TestBase): def test_set_thread_pool_size(self) -> None: download_manager = DownloadManager(services=Mock()) assert download_manager.get_thread_pool_size() > 0 pool_size = 21 download_manager.set_thread_pool_size(pool_size) assert download_manager.get_thread_pool_size() == pool_size class TestUploadManager(TestBase): def test_set_thread_pool_size(self) -> None: upload_manager = UploadManager(services=Mock()) assert upload_manager.get_thread_pool_size() > 0 pool_size = 37 upload_manager.set_thread_pool_size(pool_size) assert upload_manager.get_thread_pool_size() == pool_size