././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4248548 parfive-2.0.2/0000755000175100001710000000000014326506417012556 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/.codecov.yaml0000644000175100001710000000011014326506373015133 0ustar00runnerdockercoverage: status: project: default: threshold: 0.2% ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4208548 parfive-2.0.2/.github/0000755000175100001710000000000014326506417014116 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/.github/FUNDING.yml0000644000175100001710000000012014326506373015725 0ustar00runnerdocker# These are supported funding model platforms github: Cadair liberapay: Cadair ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/.github/release-drafter.yml0000644000175100001710000000101514326506373017704 0ustar00runnerdockername-template: 'v$NEXT_MINOR_VERSION' tag-template: 'v$NEXT_MINOR_VERSION' categories: - title: 'Breaking Changes' labels: - 'breaking' - title: 'Enhancements' labels: - 'enhancement' - title: 'Bug Fixes' labels: - 'bug' - title: 'Documentation and code quality' labels: - 'documentation' - title: 'Misc/Internal Changes' labels: - 'misc' exclude-labels: - "no changelog entry" change-template: '- $TITLE @$AUTHOR (#$NUMBER)' template: | ## Changes $CHANGES ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4208548 parfive-2.0.2/.github/workflows/0000755000175100001710000000000014326506417016153 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/.github/workflows/ci_workflows.yml0000644000175100001710000000160714326506373021413 0ustar00runnerdockername: CI on: push: branches: - 'main' - '*.*' - '!*backport*' tags: - 'v*' - '!*dev*' - '!*pre*' - '!*post*' pull_request: # Allow manual runs through the web UI workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: tests: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: envs: | - linux: mypy - linux: py38-conda - linux: py310 - linux: py311 - linux: py39 - windows: py38 - macos: py37 coverage: 'codecov' publish: needs: tests uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1 with: test_extras: tests,ftp test_command: pytest --pyargs parfive secrets: pypi_token: ${{ secrets.pypi_token }} ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/.github/workflows/release-drafter.yml0000644000175100001710000000043414326506373021745 0ustar00runnerdockername: Release Drafter on: push: branches: - main jobs: update_release_draft: permissions: contents: write runs-on: ubuntu-latest steps: - uses: release-drafter/release-drafter@v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/.gitignore0000644000175100001710000000101414326506373014543 0ustar00runnerdocker*.py[cod] .eggs/** # C extensions *.so # Packages *.egg *.egg-info dist build eggs parts bin var sdist develop-eggs .installed.cfg lib lib64 __pycache__ # Installer logs pip-log.txt # Unit test / coverage reports .coverage .tox nosetests.xml # Translations *.mo # Mr Developer .mr.developer.cfg .project .pydevproject docs/_build/ docs/api/ htmlcov/ .vscode/ .history pip-wheel-metadata/ parfive/tests/.ipynb_checkpoints/ parfive/tests/predicted-sunspot-radio-flux.txt parfive/_version.py coverage.xml *undo-tree* ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/.pre-commit-config.yaml0000644000175100001710000000113214326506373017035 0ustar00runnerdocker# See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: - id: check-ast - id: check-case-conflict - id: trailing-whitespace - id: check-yaml - id: debug-statements - id: check-added-large-files - id: end-of-file-fixer - id: mixed-line-ending - repo: https://github.com/PyCQA/isort rev: 5.10.1 hooks: - id: isort args: ['--sp','setup.cfg'] - repo: https://github.com/psf/black rev: 22.10.0 hooks: - id: black ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/.readthedocs.yml0000644000175100001710000000064114326506373015646 0ustar00runnerdocker# .readthedocs.yml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details version: 2 build: os: ubuntu-22.04 tools: python: "3.10" apt_packages: - graphviz sphinx: builder: html configuration: docs/conf.py fail_on_warning: true python: install: - method: pip path: . extra_requirements: - docs - ftp ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/LICENSE0000644000175100001710000000204714326506373013567 0ustar00runnerdockerCopyright (c) 2017-2020 Stuart Mumford Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4248548 parfive-2.0.2/PKG-INFO0000644000175100001710000000700214326506417013652 0ustar00runnerdockerMetadata-Version: 2.1 Name: parfive Version: 2.0.2 Summary: A HTTP and FTP parallel file downloader. Home-page: https://parfive.readthedocs.io/ Author: "Stuart Mumford" Author-email: "stuart@cadair.com" License: MIT Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Requires-Python: >=3.7 Provides-Extra: ftp Provides-Extra: tests Provides-Extra: docs License-File: LICENSE Parfive ======= .. image:: https://img.shields.io/pypi/v/parfive.svg :target: https://pypi.python.org/pypi/parfive :alt: Latest PyPI version A parallel file downloader using asyncio. parfive can handle downloading multiple files in parallel as well as downloading each file in a number of chunks. Usage ----- .. image:: https://asciinema.org/a/EuALahgkiicWHGmrfFsZSLz81.svg :alt: asciicast demo of parfive :target: https://asciinema.org/a/EuALahgkiicWHGmrfFsZSLz81 parfive works by creating a downloader object, appending files to it and then running the download. parfive has a synchronous API, but uses asyncio to paralellise downloading the files. A simple example is:: from parfive import Downloader dl = Downloader() dl.enqueue_file("http://data.sunpy.org/sample-data/predicted-sunspot-radio-flux.txt", path="./") files = dl.download() Parfive also bundles a CLI. The following example will download the two files concurrently.:: $ parfive 'http://212.183.159.230/5MB.zip' 'http://212.183.159.230/10MB.zip' $ parfive --help usage: parfive [-h] [--max-conn MAX_CONN] [--overwrite] [--no-file-progress] [--directory DIRECTORY] [--print-filenames] URLS [URLS ...] Parfive, the python asyncio based downloader positional arguments: URLS URLs of files to be downloaded. optional arguments: -h, --help show this help message and exit --max-conn MAX_CONN Number of maximum connections. --overwrite Overwrite if the file exists. --no-file-progress Show progress bar for each file. --directory DIRECTORY Directory to which downloaded files are saved. --print-filenames Print successfully downloaded files's names to stdout. Results ^^^^^^^ ``parfive.Downloader.download`` returns a ``parfive.Results`` object, which is a list of the filenames that have been downloaded. It also tracks any files which failed to download. Handling Errors ^^^^^^^^^^^^^^^ If files fail to download, the urls and the response from the server are stored in the ``Results`` object returned by ``parfive.Downloader``. These can be used to inform users about the errors. (Note, the progress bar will finish in an incomplete state if a download fails, i.e. it will show ``4/5 Files Downloaded``). The ``Results`` object is a list with an extra attribute ``errors``, this property returns a list of named tuples, where these named tuples contains the ``.url`` and the ``.response``, which is a ``aiohttp.ClientResponse`` or a ``aiohttp.ClientError`` object. Installation ------------ parfive is available on PyPI, you can install it with pip:: pip install parfive or if you want to use FTP downloads:: pip install parfive[ftp] Requirements ^^^^^^^^^^^^ - Python 3.7 or above - aiohttp - tqdm - aioftp (for downloads over FTP) Licence ------- MIT Licensed Authors ------- `parfive` was written by `Stuart Mumford `__. ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/README.rst0000644000175100001710000000570414326506373014254 0ustar00runnerdockerParfive ======= .. image:: https://img.shields.io/pypi/v/parfive.svg :target: https://pypi.python.org/pypi/parfive :alt: Latest PyPI version A parallel file downloader using asyncio. parfive can handle downloading multiple files in parallel as well as downloading each file in a number of chunks. Usage ----- .. image:: https://asciinema.org/a/EuALahgkiicWHGmrfFsZSLz81.svg :alt: asciicast demo of parfive :target: https://asciinema.org/a/EuALahgkiicWHGmrfFsZSLz81 parfive works by creating a downloader object, appending files to it and then running the download. parfive has a synchronous API, but uses asyncio to paralellise downloading the files. A simple example is:: from parfive import Downloader dl = Downloader() dl.enqueue_file("http://data.sunpy.org/sample-data/predicted-sunspot-radio-flux.txt", path="./") files = dl.download() Parfive also bundles a CLI. The following example will download the two files concurrently.:: $ parfive 'http://212.183.159.230/5MB.zip' 'http://212.183.159.230/10MB.zip' $ parfive --help usage: parfive [-h] [--max-conn MAX_CONN] [--overwrite] [--no-file-progress] [--directory DIRECTORY] [--print-filenames] URLS [URLS ...] Parfive, the python asyncio based downloader positional arguments: URLS URLs of files to be downloaded. optional arguments: -h, --help show this help message and exit --max-conn MAX_CONN Number of maximum connections. --overwrite Overwrite if the file exists. --no-file-progress Show progress bar for each file. --directory DIRECTORY Directory to which downloaded files are saved. --print-filenames Print successfully downloaded files's names to stdout. Results ^^^^^^^ ``parfive.Downloader.download`` returns a ``parfive.Results`` object, which is a list of the filenames that have been downloaded. It also tracks any files which failed to download. Handling Errors ^^^^^^^^^^^^^^^ If files fail to download, the urls and the response from the server are stored in the ``Results`` object returned by ``parfive.Downloader``. These can be used to inform users about the errors. (Note, the progress bar will finish in an incomplete state if a download fails, i.e. it will show ``4/5 Files Downloaded``). The ``Results`` object is a list with an extra attribute ``errors``, this property returns a list of named tuples, where these named tuples contains the ``.url`` and the ``.response``, which is a ``aiohttp.ClientResponse`` or a ``aiohttp.ClientError`` object. Installation ------------ parfive is available on PyPI, you can install it with pip:: pip install parfive or if you want to use FTP downloads:: pip install parfive[ftp] Requirements ^^^^^^^^^^^^ - Python 3.7 or above - aiohttp - tqdm - aioftp (for downloads over FTP) Licence ------- MIT Licensed Authors ------- `parfive` was written by `Stuart Mumford `__. ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4208548 parfive-2.0.2/docs/0000755000175100001710000000000014326506417013506 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/docs/Makefile0000644000175100001710000000110514326506373015144 0ustar00runnerdocker# Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/docs/conf.py0000644000175100001710000000746514326506373015022 0ustar00runnerdocker""" Configuration file for the Sphinx documentation builder. isort:skip_file """ # flake8: NOQA: E402 # -- stdlib imports ------------------------------------------------------------ from parfive import __version__ import datetime from packaging.version import Version # -- Project information ------------------------------------------------------- project = "Parfive" author = "Stuart Mumford and Contributors" copyright = "{}, {}".format(datetime.datetime.now().year, author) # The full version, including alpha/beta/rc tags release = __version__ parfive_version = Version(__version__) is_release = not (parfive_version.is_prerelease or parfive_version.is_devrelease) # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ "sphinx.ext.autodoc", "sphinx.ext.coverage", "sphinx.ext.doctest", "sphinx.ext.inheritance_diagram", "sphinx.ext.intersphinx", "sphinx.ext.mathjax", "sphinx.ext.napoleon", "sphinx.ext.todo", "sphinx.ext.viewcode", "sphinx_autodoc_typehints", # must be loaded after napoleon "sphinx_automodapi.automodapi", "sphinx_automodapi.smart_resolver", "sphinx_contributors", ] # Add any paths that contain templates here, relative to this directory. # templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. html_extra_path = ["robots.txt"] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: source_suffix = ".rst" # The master toctree document. master_doc = "index" # The reST default role (used for this markup: `text`) to use for all # documents. Set to the "smart" one. default_role = "obj" # Disable having a separate return type row napoleon_use_rtype = False # Disable google style docstrings napoleon_google_docstring = False # Type Hint Config typehints_fully_qualified = False typehints_use_rtype = napoleon_use_rtype typehints_defaults = "comma" # -- Options for intersphinx extension ----------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { "https://docs.python.org/": None, "https://docs.aiohttp.org/en/stable": None, "https://aioftp.readthedocs.io/": None, } # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = "sphinx_book_theme" html_theme_options = { "home_page_in_toc": True, "repository_url": "https://github.com/Cadair/parfive", "use_repository_button": True, "use_issues_button": True, "use_download_button": False, } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["static"] html_css_files = [ "css/contributors.css", ] # Render inheritance diagrams in SVG graphviz_output_format = "svg" graphviz_dot_args = [ "-Nfontsize=10", "-Nfontname=Helvetica Neue, Helvetica, Arial, sans-serif", "-Efontsize=10", "-Efontname=Helvetica Neue, Helvetica, Arial, sans-serif", "-Gfontsize=10", "-Gfontname=Helvetica Neue, Helvetica, Arial, sans-serif", ] ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/docs/index.rst0000644000175100001710000001344614326506373015360 0ustar00runnerdocker.. currentmodule:: parfive .. _parfive: ======= Parfive ======= Parfive is a small library for downloading files, its objective is to provide a simple API for queuing files for download and then providing excellent feedback to the user about the in progress downloads. It also aims to provide a clear interface for inspecting any failed downloads. The parfive package was motivated by the needs of `SunPy's `__ ``net`` submodule, but should be generally applicable to anyone who wants a user friendly way of downloading multiple files in parallel. Parfive uses asyncio to support downloading multiple files in parallel, and to support downloading a single file in multiple parallel chunks. Parfive supports downloading files over either HTTP or FTP using `aiohttp `__ and `aioftp `__ (``aioftp`` is an optional dependency, which does not need to be installed to download files over HTTP). Parfive provides both a function and coroutine interface, so that it can be used from both synchronous and asynchronous code. It also has opt-in support for using `aiofiles `__ to write downloaded data to disk using a separate thread pool, which may be useful if you are using parfive from within an asyncio application. Installation ------------ parfive can be installed via pip:: pip install parfive or with FTP support:: pip install parfive[ftp] or with conda from conda-forge:: conda install -c conda-forge parfive or from `GitHub `__. Usage ----- Parfive works by creating a downloader object, queuing downloads with it and then running the download. A simple example is:: from parfive import Downloader dl = Downloader() dl.enqueue_file("http://data.sunpy.org/sample-data/predicted-sunspot-radio-flux.txt", path="./") files = dl.download() It's also possible to download a list of URLs to a single destination using the `Downloader.simple_download ` method:: from parfive import Downloader files = Downloader.simple_download(['http://212.183.159.230/5MB.zip' 'http://212.183.159.230/10MB.zip'], path="./") Parfive also bundles a CLI. The following example will download the two files concurrently:: $ parfive 'http://212.183.159.230/5MB.zip' 'http://212.183.159.230/10MB.zip' $ parfive --help usage: parfive [-h] [--max-conn MAX_CONN] [--overwrite] [--no-file-progress] [--directory DIRECTORY] [--print-filenames] URLS [URLS ...] Parfive, the python asyncio based downloader positional arguments: URLS URLs of files to be downloaded. optional arguments: -h, --help show this help message and exit --max-conn MAX_CONN Number of maximum connections. --overwrite Overwrite if the file exists. --no-file-progress Show progress bar for each file. --directory DIRECTORY Directory to which downloaded files are saved. --print-filenames Print successfully downloaded files's names to stdout. Options and Customisation ------------------------- Parfive aims to support as many use cases as possible, and therefore has a number of options. There are two main points where you can customise the behaviour of the downloads, in the initialiser to `parfive.Downloader` or when adding a URL to the download queue with `~parfive.Downloader.enqueue_file`. The arguments to the ``Downloader()`` constructor affect all files transferred, and the arguments to ``enqueue_file()`` apply to only that file. By default parfive will transfer 5 files in parallel and, if supported by the remote server, chunk those files and download 5 chunks simultaneously. This behaviour is controlled by the ``max_conn=`` and ``max_splits=`` keyword arguments. Further configuration of the ``Downloader`` instance is done by passing in a `parfive.SessionConfig` object as the ``config=`` keyword argument to ``Downloader()``. See the documentation of that class for more details. Keyword arguments to `~parfive.Downloader.enqueue_file` are passed through to either `aiohttp.ClientSession.get` for HTTP downloads or `aioftp.Client` for FTP downloads. This gives you many per-file options such as headers, authentication, ssl options etc. Parfive API ----------- .. automodapi:: parfive :no-heading: :no-main-docstr: Environment Variables --------------------- Parfive reads the following environment variables, note that as of version 2.0 all environment variables are read at the point where the ``Downloader()`` class is instantiated. * ``PARFIVE_SINGLE_DOWNLOAD`` - If set to ``"True"`` this variable sets ``max_conn`` and ``max_splits`` to one; meaning that no parallelisation of the downloads will occur. * ``PARFIVE_DISABLE_RANGE`` - If set to ``"True"`` this variable will set ``max_splits`` to one; meaning that each file downloaded will only have one concurrent connection, although multiple files may be downloaded simultaneously. * ``PARFIVE_OVERWRITE_ENABLE_AIOFILES`` - If set to ``"True"`` and aiofiles is installed in the system, aiofiles will be used to write files to disk. * ``PARFIVE_DEBUG`` - If set to ``"True"`` will configure the built-in Python logger to log to stderr and set parfive, aiohttp and aioftp to debug levels. * ``PARFIVE_HIDE_PROGESS`` - If set to ``"True"`` no progress bars will be shown. * ``PARFIVE_TOTAL_TIMEOUT`` - Overrides the default aiohttp ``total`` timeout value (unless set in Python). * ``PARFIVE_SOCK_READ_TIMEOUT`` - Overrides the default aiohttp ``sock_read`` timeout value (unless set in Python). Contributors ------------ .. contributors:: Cadair/parfive :avatars: :exclude: pre-commit-ci[bot] :order: ASC Changelog --------- See `GitHub Releases `__ for the release history and changelog. ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/docs/robots.txt0000644000175100001710000000032014326506373015553 0ustar00runnerdockerUser-agent: * Allow: /*/latest/ Allow: /en/latest/ # Fallback for bots that don't understand wildcards Allow: /*/stable/ Allow: /en/stable/ # Fallback for bots that don't understand wildcards Disallow: / ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4208548 parfive-2.0.2/docs/static/0000755000175100001710000000000014326506417014775 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4248548 parfive-2.0.2/docs/static/css/0000755000175100001710000000000014326506417015565 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/docs/static/css/contributors.css0000644000175100001710000000061414326506373021036 0ustar00runnerdocker.sphinx-contributors img { border-radius: 50%; } .sphinx-contributors_list { padding-left: 0; } .sphinx-contributors_list__item { padding-right: 0.75em; padding-left: 0.75em; } .sphinx-contributors--avatars .sphinx-contributors_contributor__image { max-width: 100px; } .sphinx-contributors_contributor { width: initial; } .sphinx-contributors { width: initial; } ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4248548 parfive-2.0.2/parfive/0000755000175100001710000000000014326506417014212 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/__init__.py0000644000175100001710000000111014326506373016315 0ustar00runnerdocker""" ******* parfive ******* A parallel file downloader using asyncio. * Documentation: https://parfive.readthedocs.io/en/stable/ * Source code: https://github.com/Cadair/parfive """ import logging as _logging from .config import SessionConfig from .downloader import Downloader from .results import Results __all__ = ["SessionConfig", "Downloader", "Results", "log", "__version__"] try: from ._version import version as __version__ except ImportError: print("Version not found, please reinstall parfive.") __version__ = "unknown" log = _logging.getLogger("parfive") ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878735.0 parfive-2.0.2/parfive/_version.py0000644000175100001710000000026014326506417016406 0ustar00runnerdocker# coding: utf-8 # file generated by setuptools_scm # don't change, don't track in version control __version__ = version = '2.0.2' __version_tuple__ = version_tuple = (2, 0, 2) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/config.py0000644000175100001710000001641214326506373016036 0ustar00runnerdockerimport os import platform import warnings from typing import Dict, Union, Callable, Optional try: from typing import Literal # Added in Python 3.8 except ImportError: from typing_extensions import Literal # type: ignore from dataclasses import InitVar, field, dataclass import aiohttp import parfive from parfive.utils import ParfiveUserWarning __all__ = ["DownloaderConfig", "SessionConfig"] def _default_headers(): return { "User-Agent": f"parfive/{parfive.__version__}" f" aiohttp/{aiohttp.__version__}" f" python/{platform.python_version()}" } def _default_aiohttp_session(config: "SessionConfig") -> aiohttp.ClientSession: """ The aiohttp session with the kwargs stored by this config. Notes ----- `aiohttp.ClientSession` expects to be instantiated in a asyncio context where it can get a running loop. """ return aiohttp.ClientSession(headers=config.headers) @dataclass class EnvConfig: """ Configuration read from environment variables. """ # Session scoped env vars serial_mode: bool = field(default=False, init=False) disable_range: bool = field(default=False, init=False) hide_progress: bool = field(default=False, init=False) debug_logging: bool = field(default=False, init=False) timeout_total: float = field(default=0, init=False) timeout_sock_read: float = field(default=90, init=False) override_use_aiofiles: bool = field(default=False, init=False) def __post_init__(self): self.serial_mode = "PARFIVE_SINGLE_DOWNLOAD" in os.environ self.disable_range = "PARFIVE_DISABLE_RANGE" in os.environ self.hide_progress = "PARFIVE_HIDE_PROGRESS" in os.environ self.debug_logging = "PARFIVE_DEBUG" in os.environ self.timeout_total = float(os.environ.get("PARFIVE_TOTAL_TIMEOUT", 0)) self.timeout_sock_read = float(os.environ.get("PARFIVE_SOCK_READ_TIMEOUT", 90)) self.override_use_aiofiles = "PARFIVE_OVERWRITE_ENABLE_AIOFILES" in os.environ @dataclass class SessionConfig: """ Configuration options for `parfive.Downloader`. """ http_proxy: Optional[str] = None """ The URL of a proxy to use for HTTP requests. Will default to the value of the ``HTTP_PROXY`` env var. """ https_proxy: Optional[str] = None """ The URL of a proxy to use for HTTPS requests. Will default to the value of the ``HTTPS_PROXY`` env var. """ headers: Optional[Dict[str, str]] = field(default_factory=_default_headers) """ Headers to be passed to all requests made by this session. These headers are passed to the `aiohttp.ClientSession` along with ``aiohttp_session_kwargs``. The default value for headers is setting the user agent to a string with the version of parfive, aiohttp and Python. To use aiohttp's default headers set this argument to an empty dictionary. """ chunksize: float = 1024 """ The default chunksize to be used for transfers over HTTP. """ file_progress: bool = True """ If `True` (the default) a progress bar will be shown (if any progress bars are shown) for every file, in addition for one showing progress of downloading all file. """ notebook: Union[bool, None] = None """ Override automatic detection of Jupyter notebook for drawing progress bars. If `None` `tqdm` will automatically detect if it can draw rich notebook progress bars. If `False` or `True` notebook mode will be forced off or on. """ log_level: Optional[str] = None """ If not `None` configure the logger to log to stderr with this log level. """ use_aiofiles: Optional[bool] = False """ Enables using `aiofiles` to write files to disk in their own thread pool. The default value is `False`. This argument will be overridden by the ``PARFIVE_OVERWRITE_ENABLE_AIOFILES`` environment variable. If `aiofiles` can not be imported then this will be set to `False`. """ timeouts: Optional[aiohttp.ClientTimeout] = None """ The `aiohttp.ClientTimeout` object to control the timeouts used for all HTTP requests. By default the ``total`` timeout is set to `0` (never timeout) and the ``sock_read`` timeout is set to `90` seconds. These defaults can also be overridden by the ``PARFIVE_TOTAL_TIMEOUT`` and ``PARFIVE_SOCK_READ_TIMEOUT`` environment variables. """ aiohttp_session_generator: Optional[Callable[["SessionConfig"], aiohttp.ClientSession]] = None """ A function to override the generation of the `aiohttp.ClientSession` object. Due to the fact that this session needs to be instantiated inside the asyncio context this option is a function. This function takes one argument which is the instance of this ``SessionConfig`` class. It is expected that you pass the ``.headers`` attribute of the config instance through to the ``headers=`` keyword argument of the session you instantiate. """ env: EnvConfig = field(default_factory=EnvConfig) @staticmethod def _aiofiles_importable(): try: import aiofiles except ImportError: return False return True def _compute_aiofiles(self, use_aiofiles): use_aiofiles = use_aiofiles or self.env.override_use_aiofiles if use_aiofiles and not self._aiofiles_importable(): warnings.warn( "Can not use aiofiles even though use_aiofiles is set to True as aiofiles can not be imported.", ParfiveUserWarning, ) use_aiofiles = False return use_aiofiles def __post_init__(self): if self.timeouts is None: timeouts = { "total": self.env.timeout_total, "sock_read": self.env.timeout_sock_read, } self.timeouts = aiohttp.ClientTimeout(**timeouts) if self.http_proxy is None: self.http_proxy = os.environ.get("HTTP_PROXY", None) if self.https_proxy is None: self.https_proxy = os.environ.get("HTTPS_PROXY", None) if self.use_aiofiles is not None: self.use_aiofiles = self._compute_aiofiles(self.use_aiofiles) if self.env.debug_logging: self.log_level = "DEBUG" @dataclass class DownloaderConfig: """ Hold all downloader session state. """ max_conn: int = 5 max_splits: int = 5 progress: bool = True overwrite: Union[bool, Literal["unique"]] = False config: Optional[SessionConfig] = field(default_factory=SessionConfig) env: EnvConfig = field(default_factory=EnvConfig) def __post_init__(self): if self.config is None: self.config = SessionConfig() self.max_conn = 1 if self.env.serial_mode else self.max_conn self.max_splits = 1 if self.env.serial_mode or self.env.disable_range else self.max_splits self.progress = False if self.env.hide_progress else self.progress if self.progress is False: self.file_progress = False def __getattr__(self, __name: str): return getattr(self.config, __name) def aiohttp_client_session(self): if self.config.aiohttp_session_generator is None: return _default_aiohttp_session(self.config) return self.config.aiohttp_session_generator(self.config) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/conftest.py0000644000175100001710000000103014326506373016404 0ustar00runnerdockerfrom functools import partial import pytest from parfive.tests.localserver import MultiPartTestServer, SimpleTestServer, error_on_nth_request @pytest.fixture def testserver(): server = SimpleTestServer(callback=partial(error_on_nth_request, 2)) server.start_server() yield server server.stop_server() @pytest.fixture def multipartserver(): """ A server that can handle multi-part file downloads """ server = MultiPartTestServer() server.start_server() yield server server.stop_server() ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/downloader.py0000644000175100001710000007407514326506373016740 0ustar00runnerdockerimport os import signal import asyncio import logging import pathlib import contextlib import urllib.parse from typing import Union, Callable, Optional from functools import reduce try: from typing import Literal # Added in Python 3.8 except ImportError: from typing_extensions import Literal # type: ignore from functools import partial import aiohttp from tqdm import tqdm as tqdm_std from tqdm.auto import tqdm as tqdm_auto import parfive from .config import DownloaderConfig, SessionConfig from .results import Results from .utils import ( FailedDownload, MultiPartDownloadError, Token, _QueueList, cancel_task, default_name, get_filepath, get_ftp_size, get_http_size, remove_file, run_task_in_thread, ) try: import aioftp except ImportError: # pragma: nocover aioftp = None __all__ = ["Downloader"] class Downloader: """ Download files in parallel. Parameters ---------- max_conn The number of parallel download slots. max_splits The maximum number of splits to use to download a file (server dependent). progress If `True` show a main progress bar showing how many of the total files have been downloaded. If `False`, no progress bars will be shown at all. overwrite Determine how to handle downloading if a file already exists with the same name. If `False` the file download will be skipped and the path returned to the existing file, if `True` the file will be downloaded and the existing file will be overwritten, if `'unique'` the filename will be modified to be unique. config A config object containing more complex settings for this ``Downloader`` instance. """ def __init__( self, max_conn: int = 5, max_splits: int = 5, progress: bool = True, overwrite: Union[bool, Literal["unique"]] = False, config: SessionConfig = None, ): self.config = DownloaderConfig( max_conn=max_conn, max_splits=max_splits, progress=progress, overwrite=overwrite, config=config, ) self._init_queues() # Configure progress bars self.tqdm = tqdm_auto if self.config.notebook is not None: if self.config.notebook is True: from tqdm.notebook import tqdm as tqdm_notebook self.tqdm = tqdm_notebook elif self.config.notebook is False: self.tqdm = tqdm_std else: raise ValueError( "The notebook keyword argument should be one of None, True or False." ) self._configure_logging() def _init_queues(self): # Setup queues self.http_queue = _QueueList() self.ftp_queue = _QueueList() def _generate_tokens(self): # Create a Queue with max_conn tokens queue = asyncio.Queue(maxsize=self.config.max_conn) for i in range(self.config.max_conn): queue.put_nowait(Token(i + 1)) return queue def _configure_logging(self): # pragma: no cover if self.config.log_level is None: return sh = logging.StreamHandler() sh.setLevel(self.config.log_level) formatter = logging.Formatter("%(name)s - %(levelname)s - %(message)s") sh.setFormatter(formatter) parfive.log.addHandler(sh) parfive.log.setLevel(self.config.log_level) aiohttp_logger = logging.getLogger("aiohttp.client") aioftp_logger = logging.getLogger("aioftp.client") aioftp_logger.addHandler(sh) aioftp_logger.setLevel(self.config.log_level) aiohttp_logger.addHandler(sh) aiohttp_logger.setLevel(self.config.log_level) parfive.log.debug("Configured parfive to run with debug logging...") @property def queued_downloads(self): """ The total number of files already queued for download. """ return len(self.http_queue) + len(self.ftp_queue) def enqueue_file( self, url: str, path: Union[str, os.PathLike] = None, filename: Union[str, Callable[[str, Optional[aiohttp.ClientResponse]], os.PathLike]] = None, overwrite: Union[bool, Literal["unique"]] = None, **kwargs, ): """ Add a file to the download queue. Parameters ---------- url The URL to retrieve. path The directory to retrieve the file into, if `None` defaults to the current directory. filename The filename to save the file as. Can also be a callable which takes two arguments the url and the response object from opening that URL, and returns the filename. (Note, for FTP downloads the response will be ``None``.) If `None` the HTTP headers will be read for the filename, or the last segment of the URL will be used. overwrite Determine how to handle downloading if a file already exists with the same name. If `False` the file download will be skipped and the path returned to the existing file, if `True` the file will be downloaded and the existing file will be overwritten, if `'unique'` the filename will be modified to be unique. If `None` the value set when constructing the `~parfive.Downloader` object will be used. kwargs : `dict` Extra keyword arguments are passed to `aiohttp.ClientSession.request` or `aioftp.Client.context` depending on the protocol. """ overwrite = overwrite or self.config.overwrite if path is None and filename is None: raise ValueError("Either path or filename must be specified.") elif path is None: path = "./" path = pathlib.Path(path) filepath: Callable[[str, Optional[aiohttp.ClientResponse]], os.PathLike] if not filename: filepath = partial(default_name, path) elif callable(filename): filepath = filename else: # Define a function because get_file expects a callback def filepath(url, resp): return path / filename scheme = urllib.parse.urlparse(url).scheme if scheme in ("http", "https"): get_file = partial( self._get_http, url=url, filepath_partial=filepath, overwrite=overwrite, **kwargs ) self.http_queue.append(get_file) elif scheme == "ftp": if aioftp is None: raise ValueError("The aioftp package must be installed to download over FTP.") get_file = partial( self._get_ftp, url=url, filepath_partial=filepath, overwrite=overwrite, **kwargs ) self.ftp_queue.append(get_file) else: raise ValueError("URL must start with either 'http' or 'ftp'.") @staticmethod def _add_shutdown_signals(loop, task): if os.name == "nt": return for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(sig, task.cancel) def _run_in_loop(self, coro): """ Take a coroutine and figure out where to run it and how to cancel it. """ try: loop = asyncio.get_running_loop() except RuntimeError: loop = None # If we already have a loop and it's already running then we should # make a new loop (as we are probably in a Jupyter Notebook) should_run_in_thread = loop and loop.is_running() # If we don't already have a loop, make a new one if should_run_in_thread or loop is None: loop = asyncio.new_event_loop() # Wrap up the coroutine in a task so we can cancel it later task = loop.create_task(coro) # Add handlers for shutdown signals self._add_shutdown_signals(loop, task) # Execute the task if should_run_in_thread: return run_task_in_thread(loop, task) return loop.run_until_complete(task) async def run_download(self): """ Download all files in the queue. Returns ------- `parfive.Results` A list of files downloaded. """ tasks = set() with self._get_main_pb(self.queued_downloads) as main_pb: try: if len(self.http_queue): tasks.add(asyncio.create_task(self._run_http_download(main_pb))) if len(self.ftp_queue): tasks.add(asyncio.create_task(self._run_ftp_download(main_pb))) dl_results = await asyncio.gather(*tasks, return_exceptions=True) except asyncio.CancelledError: for task in tasks: task.cancel() dl_results = await asyncio.gather(*tasks, return_exceptions=True) finally: results_obj = self._format_results(dl_results, main_pb) return results_obj def _format_results(self, retvals, main_pb): # Squash all nested lists into a single flat list if retvals and isinstance(retvals[0], list): retvals = list(reduce(list.__add__, retvals)) errors = sum([isinstance(i, FailedDownload) for i in retvals]) if errors: total_files = self.queued_downloads message = f"{errors}/{total_files} files failed to download. Please check `.errors` for details" if main_pb: main_pb.write(message) else: parfive.log.info(message) results = Results() # Iterate through the results and store any failed download errors in # the errors list of the results object. for res in retvals: if isinstance(res, FailedDownload): results.add_error(res.filepath_partial, res.url, res.exception) parfive.log.info( "%s failed to download with exception\n" "%s", res.url, res.exception ) elif isinstance(res, Exception): raise res else: results.append(res) return results def download(self): """ Download all files in the queue. Returns ------- `parfive.Results` A list of files downloaded. Notes ----- This is a synchronous version of `~parfive.Downloader.run_download`, an `asyncio` event loop will be created to run the download (in it's own thread if a loop is already running). """ return self._run_in_loop(self.run_download()) def retry(self, results: Results): """ Retry any failed downloads in a results object. .. note:: This will start a new event loop. Parameters ---------- results : `parfive.Results` A previous results object, the ``.errors`` property will be read and the downloads retried. Returns ------- `parfive.Results` A modified version of the input ``results`` with all the errors from this download attempt and any new files appended to the list of file paths. """ # Reset the queues self._init_queues() for err in results.errors: self.enqueue_file(err.url, filename=err.filepath_partial) new_res = self.download() results += new_res results._errors = new_res._errors return results @classmethod def simple_download(cls, urls, *, path="./", overwrite=None): """ Download a series of URLs to a single destination. Parameters ---------- urls : iterable A sequence of URLs to download. path : `pathlib.Path`, optional The destination directory for the downloaded files. Defaults to the current directory. overwrite: `bool`, optional Overwrite the files at the destination directory. If `False` the URL will not be downloaded if a file with the corresponding filename already exists. Returns ------- `parfive.Results` A list of files downloaded. """ dl = cls() for url in urls: dl.enqueue_file(url, path=path, overwrite=overwrite) return dl.download() def _get_main_pb(self, total): """ Return the tqdm instance if we want it, else return a contextmanager that just returns None. """ if self.config.progress: return self.tqdm(total=total, unit="file", desc="Files Downloaded", position=0) else: return contextlib.contextmanager(lambda: iter([None]))() async def _run_http_download(self, main_pb): async with self.config.aiohttp_client_session() as session: futures = await self._run_from_queue( self.http_queue.generate_queue(), self._generate_tokens(), main_pb, session=session, ) try: # Wait for all the coroutines to finish done, _ = await asyncio.wait(futures) except asyncio.CancelledError: for task in futures: task.cancel() return await asyncio.gather(*futures, return_exceptions=True) async def _run_ftp_download(self, main_pb): futures = await self._run_from_queue( self.ftp_queue.generate_queue(), self._generate_tokens(), main_pb, ) try: # Wait for all the coroutines to finish done, _ = await asyncio.wait(futures) except asyncio.CancelledError: for task in futures: task.cancel() return await asyncio.gather(*futures, return_exceptions=True) async def _run_from_queue(self, queue, tokens, main_pb, *, session=None): futures = [] try: while not queue.empty(): get_file = await queue.get() token = await tokens.get() file_pb = self.tqdm if self.config.file_progress else False future = asyncio.create_task(get_file(session, token=token, file_pb=file_pb)) def callback(token, future, main_pb): try: tokens.put_nowait(token) # Update the main progressbar if main_pb and not future.exception(): main_pb.update(1) except asyncio.CancelledError: return future.add_done_callback(partial(callback, token, main_pb=main_pb)) futures.append(future) except asyncio.CancelledError: for task in futures: task.cancel() return futures async def _get_http( self, session, *, url, filepath_partial, chunksize=None, file_pb=None, token, overwrite, max_splits=None, **kwargs, ): """ Read the file from the given url into the filename given by ``filepath_partial``. Parameters ---------- session : `aiohttp.ClientSession` The `aiohttp.ClientSession` to use to retrieve the files. url : `str` The url to retrieve. filepath_partial : `callable` A function to call which returns the filepath to save the url to. Takes two arguments ``resp, url``. chunksize : `int` The number of bytes to read into the file at a time. file_pb : `tqdm.tqdm` or `False` Should progress bars be displayed for each file downloaded. token : `parfive.downloader.Token` A token for this download slot. overwrite : `bool` Overwrite the file if it already exists. max_splits: `int`, optional Number of maximum concurrent connections per file. kwargs : `dict` Extra keyword arguments are passed to `aiohttp.ClientSession.get`. Returns ------- `str` The name of the file saved. """ if chunksize is None: chunksize = 1024 if max_splits is None: max_splits = self.config.max_splits # Define filepath and writer here as we use them in the except block filepath = writer = None tasks = [] try: scheme = urllib.parse.urlparse(url).scheme if scheme == "http": kwargs["proxy"] = self.config.http_proxy elif scheme == "https": kwargs["proxy"] = self.config.https_proxy async with session.get(url, timeout=self.config.timeouts, **kwargs) as resp: parfive.log.debug( "%s request made to %s with headers=%s", resp.request_info.method, resp.request_info.url, resp.request_info.headers, ) parfive.log.debug( "%s Response received from %s with headers=%s", resp.status, resp.request_info.url, resp.headers, ) if resp.status < 200 or resp.status >= 300: raise FailedDownload(filepath_partial, url, resp) else: filepath, skip = get_filepath(filepath_partial(resp, url), overwrite) if skip: parfive.log.debug( "File %s already exists and overwrite is False; skipping download.", filepath, ) return str(filepath) if callable(file_pb): file_pb = file_pb( position=token.n, unit="B", unit_scale=True, desc=filepath.name, leave=False, total=get_http_size(resp), ) else: file_pb = None # This queue will contain the downloaded chunks and their offsets # as tuples: (offset, chunk) downloaded_chunk_queue = asyncio.Queue() writer = asyncio.create_task( self._write_worker(downloaded_chunk_queue, file_pb, filepath) ) if ( not self.config.env.disable_range and max_splits and resp.headers.get("Accept-Ranges", None) == "bytes" and "Content-length" in resp.headers ): content_length = int(resp.headers["Content-length"]) split_length = max(1, content_length // max_splits) ranges = [ [start, start + split_length] for start in range(0, content_length, split_length) ] # let the last part download everything ranges[-1][1] = "" for _range in ranges: tasks.append( asyncio.create_task( self._http_download_worker( session, url, chunksize, _range, downloaded_chunk_queue, **kwargs, ) ) ) else: tasks.append( asyncio.create_task( self._http_download_worker( session, url, chunksize, None, downloaded_chunk_queue, **kwargs, ) ) ) # Close the initial request here before we start transferring data. # run all the download workers await asyncio.gather(*tasks) # join() waits till all the items in the queue have been processed await downloaded_chunk_queue.join() return str(filepath) except (Exception, asyncio.CancelledError) as e: for task in tasks: task.cancel() # We have to cancel the writer here before we try and remove the # file so it's closed (otherwise windows gets angry). if writer is not None: await cancel_task(writer) # Set writer to None so we don't cancel it twice. writer = None # If filepath is None then the exception occurred before the request # computed the filepath, so we have no file to cleanup if filepath is not None: remove_file(filepath) raise FailedDownload(filepath_partial, url, e) finally: if writer is not None: writer.cancel() if isinstance(file_pb, self.tqdm): file_pb.close() async def _write_worker(self, queue, file_pb, filepath): """ Worker for writing the downloaded chunk to the file. The downloaded chunk is put into a asyncio Queue by a download worker. This worker gets the chunk from the queue and write it to the file using the specified offset of the chunk. Parameters ---------- queue: `asyncio.Queue` Queue for chunks file_pb : `tqdm.tqdm` or `False` Should progress bars be displayed for each file downloaded. filepath: `pathlib.Path` Path to the which the file should be downloaded. """ if self.config.use_aiofiles: await self._async_write_worker(queue, file_pb, filepath) else: await self._blocking_write_worker(queue, file_pb, filepath) async def _async_write_worker(self, queue, file_pb, filepath): import aiofiles async with aiofiles.open(filepath, mode="wb") as f: while True: offset, chunk = await queue.get() await f.seek(offset) await f.write(chunk) await f.flush() # Update the progressbar for file if file_pb is not None: file_pb.update(len(chunk)) queue.task_done() async def _blocking_write_worker(self, queue, file_pb, filepath): with open(filepath, "wb") as f: while True: offset, chunk = await queue.get() f.seek(offset) f.write(chunk) f.flush() # Update the progressbar for file if file_pb is not None: file_pb.update(len(chunk)) queue.task_done() async def _http_download_worker(self, session, url, chunksize, http_range, queue, **kwargs): """ Worker for downloading chunks from http urls. This function downloads the chunk from the specified http range and puts the chunk in the asyncio Queue. If no range is specified, then the whole file is downloaded via chunks and put in the queue. Parameters ---------- session : `aiohttp.ClientSession` The `aiohttp.ClientSession` to use to retrieve the files. url : `str` The url to retrieve. chunksize : `int` The number of bytes to read into the file at a time. http_range: (`int`, `int`) or `None` Start and end bytes of the file. In None, then no `Range` header is specified in request and the whole file will be downloaded. queue: `asyncio.Queue` Queue to put the download chunks. kwargs : `dict` Extra keyword arguments are passed to `aiohttp.ClientSession.get`. """ headers = kwargs.pop("headers", {}) if http_range: headers["Range"] = "bytes={}-{}".format(*http_range) # init offset to start of range offset, _ = http_range else: offset = 0 async with session.get(url, timeout=self.config.timeouts, headers=headers, **kwargs) as resp: parfive.log.debug( "%s request made for download to %s with headers=%s", resp.request_info.method, resp.request_info.url, resp.request_info.headers, ) parfive.log.debug( "%s Response received from %s with headers=%s", resp.status, resp.request_info.url, resp.headers, ) if resp.status < 200 or resp.status >= 300: raise MultiPartDownloadError(resp) while True: chunk = await resp.content.read(chunksize) if not chunk: break await queue.put((offset, chunk)) offset += len(chunk) async def _get_ftp( self, session=None, *, url, filepath_partial, file_pb=None, token, overwrite, **kwargs, ): """ Read the file from the given url into the filename given by ``filepath_partial``. Parameters ---------- session : `None` A placeholder for API compatibility with ``_get_http`` url : `str` The url to retrieve. filepath_partial : `callable` A function to call which returns the filepath to save the url to. Takes two arguments ``resp, url``. file_pb : `tqdm.tqdm` or `False` Should progress bars be displayed for each file downloaded. token : `parfive.downloader.Token` A token for this download slot. overwrite : `bool` Whether to overwrite the file if it already exists. kwargs : `dict` Extra keyword arguments are passed to `aioftp.Client.context`. Returns ------- `str` The name of the file saved. """ filepath = writer = None parse = urllib.parse.urlparse(url) try: async with aioftp.Client.context(parse.hostname, **kwargs) as client: parfive.log.debug("Connected to ftp server %s", parse.hostname) if parse.username and parse.password: parfive.log.debug( "Explicitly Logging in with %s:%s", parse.username, parse.password ) await client.login(parse.username, parse.password) # This has to be done before we start streaming the file: filepath, skip = get_filepath(filepath_partial(None, url), overwrite) if skip: parfive.log.debug( "File %s already exists and overwrite is False; skipping download.", filepath, ) return str(filepath) if callable(file_pb): total_size = await get_ftp_size(client, parse.path) file_pb = file_pb( position=token.n, unit="B", unit_scale=True, desc=filepath.name, leave=False, total=total_size, ) else: file_pb = None parfive.log.debug("Downloading file %s from %s", parse.path, parse.hostname) async with client.download_stream(parse.path) as stream: downloaded_chunks_queue = asyncio.Queue() download_workers = [] writer = asyncio.create_task( self._write_worker(downloaded_chunks_queue, file_pb, filepath) ) download_workers.append( asyncio.create_task( self._ftp_download_worker(stream, downloaded_chunks_queue) ) ) await asyncio.gather(*download_workers) await downloaded_chunks_queue.join() return str(filepath) except (Exception, asyncio.CancelledError) as e: if writer is not None: await cancel_task(writer) writer = None # If filepath is None then the exception occurred before the request # computed the filepath, so we have no file to cleanup if filepath is not None: remove_file(filepath) raise FailedDownload(filepath_partial, url, e) finally: # Just make sure we close the file. if writer is not None: writer.cancel() if isinstance(file_pb, self.tqdm): file_pb.close() async def _ftp_download_worker(self, stream, queue): """ Similar to `Downloader._http_download_worker`. See that function's documentation for more info. Parameters ---------- stream: `aioftp.StreamIO` Stream of the file to be downloaded. queue: `asyncio.Queue` Queue to put the download chunks. """ offset = 0 async for chunk in stream.iter_by_block(): # Write this chunk to the output file. await queue.put((offset, chunk)) offset += len(chunk) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/main.py0000644000175100001710000000537114326506373015517 0ustar00runnerdockerimport sys import argparse from parfive import Downloader, SessionConfig, __version__ def main(): args = parse_args(sys.argv[1:]) run_parfive(args) def run_parfive(args): log_level = "DEBUG" if args.verbose else None config = SessionConfig(file_progress=not args.no_file_progress, log_level=log_level) downloader = Downloader( max_conn=args.max_conn, max_splits=args.max_splits, progress=not args.no_progress, overwrite=args.overwrite, config=config, ) for url in args.urls: downloader.enqueue_file(url, path=args.directory) results = downloader.download() if args.print_filenames: for i in results: print(i) err_str = "" for err in results.errors: err_str += f"{err.url} \t {err.exception}\n" if err_str: print(err_str, file=sys.stderr) sys.exit(1) sys.exit(0) def parse_args(args): parser = argparse.ArgumentParser( description="Parfive: A parallel file downloader written in Python." ) parser.add_argument( "urls", metavar="URLS", type=str, nargs="+", help="URLs of files to be downloaded." ) parser.add_argument( "--max-conn", type=int, default=5, help="Maximum number of parallel file downloads." ) parser.add_argument( "--max-splits", type=int, default=5, help="Maximum number of parallel connections per file (only used if supported by the server).", ) parser.add_argument( "--directory", type=str, default="./", help="Directory to which downloaded files are saved." ) parser.add_argument( "--overwrite", action="store_const", const=True, default=False, help="Overwrite if the file exists.", ) parser.add_argument( "--no-progress", action="store_const", const=True, default=False, dest="no_progress", help="Show progress indicators during download.", ) parser.add_argument( "--no-file-progress", action="store_const", const=True, default=False, dest="no_file_progress", help="Show progress bar for each file.", ) parser.add_argument( "--print-filenames", action="store_const", const=True, default=False, dest="print_filenames", help="Print successfully downloaded files's names to stdout.", ) parser.add_argument( "--verbose", action="store_const", const=True, default=False, help="Log debugging output while transferring the files.", ) parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") args = parser.parse_args(args) return args ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/results.py0000644000175100001710000000537214326506373016275 0ustar00runnerdockerfrom collections import UserList, namedtuple import aiohttp from .utils import FailedDownload __all__ = ["Results"] class Error(namedtuple("error", ("filepath_partial", "url", "exception"))): def __str__(self): filepath_partial = "" if isinstance(self.filepath_partial, str): filepath_partial = f"{self.filepath_partial},\n" return filepath_partial + f"{self.url},\n{self.exception}" def __repr__(self): return f"{object.__repr__(self)}\n{self}" class Results(UserList): """ The results of a download from `parfive.Downloader.download`. This object contains the filenames of successful downloads as well as a list of any errors encountered in the `~parfive.Results.errors` property. """ def __init__(self, *args, errors=None): super().__init__(*args) self._errors = errors or list() def _get_nice_resp_repr(self, response): # This is a modified version of aiohttp.ClientResponse.__repr__ if isinstance(response, aiohttp.ClientResponse): ascii_encodable_url = str(response.url) if response.reason: ascii_encodable_reason = response.reason.encode("ascii", "backslashreplace").decode( "ascii" ) else: ascii_encodable_reason = response.reason return "".format( ascii_encodable_url, response.status, ascii_encodable_reason ) else: return repr(response) def __str__(self): out = super().__repr__() if self.errors: out += "\nErrors:\n" for error in self.errors: if isinstance(error, FailedDownload): resp = self._get_nice_resp_repr(error.exception) out += f"(url={error.url}, response={resp})\n" else: out += "({})".format(repr(error)) return out def __repr__(self): out = object.__repr__(self) out += "\n" out += str(self) return out def add_error(self, filename, url, exception): """ Add an error to the results. """ if isinstance(exception, aiohttp.ClientResponse): exception._headers = None self._errors.append(Error(filename, url, exception)) @property def errors(self): """ A list of errors encountered during the download. The errors are represented as a tuple containing ``(filepath, url, exception)`` where ``filepath`` is a function for generating a filepath, ``url`` is the url to be downloaded and ``exception`` is the error raised during download. """ return self._errors ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4248548 parfive-2.0.2/parfive/tests/0000755000175100001710000000000014326506417015354 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/tests/__init__.py0000644000175100001710000000000014326506373017454 0ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/tests/localserver.py0000644000175100001710000000615514326506373020257 0ustar00runnerdockerimport abc from pytest_localserver.http import WSGIServer class BaseTestServer(abc.ABC): """ A pytest-localserver server which allows you to customise it's responses. Parameters ---------- callback A callable with signature ``(request_number, environ, start_response)``. If the callback returns anything other than `None` it is assumed that the callback has handled the WSGI request. If the callback returns `None` then `default_request_handler` is returned which will handle the WSGI request. """ def __init__(self, callback=None): self.requests = [] self.server = WSGIServer(application=self.request_handler) self.callback = callback self.request_number = 0 def callback_handler(self, environ, start_response): if self.callback is not None: return self.callback(self.request_number, environ, start_response) def request_handler(self, environ, start_response): self.requests.append(environ) callback_return = self.callback_handler(environ, start_response) self.request_number += 1 if callback_return: return callback_return return self.default_request_handler(environ, start_response) @abc.abstractmethod def default_request_handler(self, environ, start_response): return def start_server(self): self.server.start() def stop_server(self): self.server.stop() @property def url(self): return self.server.url class SimpleTestServer(BaseTestServer): def default_request_handler(self, environ, start_response): status = "200 OK" response_headers = [ ("Content-type", "text/plain"), ("Content-Disposition", "attachment; filename=testfile_{self.request_number}.txt"), ] start_response(status, response_headers) return [b"Hello world!\n"] class MultiPartTestServer(BaseTestServer): def default_request_handler(self, environ, start_response): content = b"a" * 100 bytes_end = content_length = len(content) bytes_start = 0 http_range = environ.get("HTTP_RANGE", None) if http_range: http_range = http_range.split("bytes=")[1] bytes_start = int(http_range.split("-")[0]) bytes_end = http_range.split("-")[1] if not bytes_end: bytes_end = content_length bytes_end = int(bytes_end) content_length = bytes_end - bytes_start status = "200 OK" response_headers = [ ("Content-type", "text/plain"), ("Content-Length", content_length), ("Accept-Ranges", "bytes"), ("Content-Disposition", "attachment; filename=testfile.txt"), ] start_response(status, response_headers) part = content[bytes_start:bytes_end] return [part] def error_on_nth_request(n, i, environ, start_response): if i == n: status = "404" response_headers = [("Content-type", "text/plain")] start_response(status, response_headers) return [b""] ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/tests/simple_download_test.ipynb0000644000175100001710000002402714326506373022644 0ustar00runnerdocker{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import parfive" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "dl = parfive.Downloader()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "dl.enqueue_file(\"http://data.sunpy.org/sample-data/predicted-sunspot-radio-flux.txt\", path=\"./\")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "nbreg": { "diff_ignore": [ "/outputs/0/data/" ] } }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "db630dac73074c248fb6ecc163e1fd75", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(FloatProgress(value=0.0, description='Files Downloaded', max=1.0, style=ProgressStyle(descripti…" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "files = dl.download()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "\n", "['predicted-sunspot-radio-flux.txt']" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "files" ] } ], "metadata": { "celltoolbar": "Edit Metadata", "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" }, "nbreg": { "diff_ignore": [ "/metadata/widgets" ] }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": { "0cc46587e5314f8a95e5373eed4af735": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "Files Downloaded: 100%", "description_tooltip": null, "layout": "IPY_MODEL_d3f4f43178564251aa135d08e87f47db", "max": 1, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_ef5d56b1823d442c829e821516cdbff0", "value": 1 } }, "4a1f3a7b8acd4d1daeebd13033a5ba05": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "658c594e4c7f44e986c5e8d346e290ef": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "720f662a1e20406ba71816e33cbdbe30": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c5a8b53e793f4b328543dd9fedd64b6f": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_658c594e4c7f44e986c5e8d346e290ef", "placeholder": "​", "style": "IPY_MODEL_4a1f3a7b8acd4d1daeebd13033a5ba05", "value": " 1/1 [00:00<00:00, 10.57file/s]" } }, "d3f4f43178564251aa135d08e87f47db": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "db630dac73074c248fb6ecc163e1fd75": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_0cc46587e5314f8a95e5373eed4af735", "IPY_MODEL_c5a8b53e793f4b328543dd9fedd64b6f" ], "layout": "IPY_MODEL_720f662a1e20406ba71816e33cbdbe30" } }, "ef5d56b1823d442c829e821516cdbff0": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial" } } }, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 4 } ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/tests/test_aiofiles.py0000644000175100001710000000307614326506373020567 0ustar00runnerdockerimport os from unittest.mock import patch import pytest import parfive from parfive import Downloader from parfive.config import SessionConfig from parfive.utils import ParfiveUserWarning @pytest.mark.parametrize("use_aiofiles", [True, False]) def test_enable_aiofiles_constructor(use_aiofiles): dl = Downloader(config=parfive.SessionConfig(use_aiofiles=use_aiofiles)) assert ( dl.config.use_aiofiles == use_aiofiles ), f"expected={use_aiofiles}, got={dl.config.use_aiofiles}" @patch.dict(os.environ, {"PARFIVE_OVERWRITE_ENABLE_AIOFILES": "some_value_to_enable_it"}) @pytest.mark.parametrize("use_aiofiles", [True, False]) def test_enable_aiofiles_env_overwrite_always_enabled(use_aiofiles): dl = Downloader(config=parfive.SessionConfig(use_aiofiles=use_aiofiles)) assert dl.config.use_aiofiles is True @patch("parfive.config.SessionConfig._aiofiles_importable", lambda self: False) def test_enable_no_aiofiles(): with pytest.warns(ParfiveUserWarning): dl = Downloader(config=parfive.SessionConfig(use_aiofiles=True)) assert dl.config.use_aiofiles is False dl = Downloader(config=parfive.SessionConfig(use_aiofiles=False)) assert dl.config.use_aiofiles is False def test_aiofiles_session_config(): c = SessionConfig(use_aiofiles=True) assert c.use_aiofiles is True @patch("parfive.config.SessionConfig._aiofiles_importable", lambda self: False) def test_aiofiles_session_config_no_aiofiles_warn(): with pytest.warns(ParfiveUserWarning): c = SessionConfig(use_aiofiles=True) assert c.use_aiofiles is False ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/tests/test_config.py0000644000175100001710000000231614326506373020235 0ustar00runnerdockerimport ssl import aiohttp import pytest from parfive.config import DownloaderConfig, SessionConfig from parfive.downloader import Downloader from parfive.utils import ParfiveFutureWarning def test_session_config_defaults(): c = SessionConfig() assert c.aiohttp_session_generator is None assert isinstance(c.timeouts, aiohttp.ClientTimeout) assert c.timeouts.total == 0 assert c.timeouts.sock_read == 90 assert c.http_proxy is None assert c.https_proxy is None assert c.chunksize == 1024 assert c.use_aiofiles is False assert isinstance(c.headers, dict) assert "User-Agent" in c.headers assert "parfive" in c.headers["User-Agent"] def test_session_config_env_defaults(): c = SessionConfig() assert c.env.serial_mode is False assert c.env.disable_range is False assert c.env.hide_progress is False assert c.env.timeout_total == 0 assert c.env.timeout_sock_read == 90 def test_ssl_context(): # Assert that the unpickalable SSL context object doesn't anger the # dataclass gods gen = lambda config: aiohttp.ClientSession(context=ssl.create_default_context()) c = SessionConfig(aiohttp_session_generator=gen) d = Downloader(config=c) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/tests/test_downloader.py0000644000175100001710000002740414326506373021133 0ustar00runnerdockerimport os import platform from pathlib import Path from unittest import mock from unittest.mock import patch import aiohttp import pytest from aiohttp import ClientTimeout import parfive from parfive.config import SessionConfig from parfive.downloader import Downloader, FailedDownload, Results, Token from parfive.utils import sha256sum skip_windows = pytest.mark.skipif(platform.system() == "Windows", reason="Windows.") def validate_test_file(f): assert len(f) == 1 assert Path(f[0]).name == "testfile.fits" assert sha256sum(f[0]) == "a1c58cd340e3bd33f94524076f1fa5cf9a7f13c59d5272a9d4bc0b5bc436d9b3" def test_setup(): dl = Downloader() assert isinstance(dl, Downloader) assert len(dl.http_queue) == 0 assert len(dl.ftp_queue) == 0 assert dl._generate_tokens().qsize() == 5 def test_download(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content( "SIMPLE = T", headers={"Content-Disposition": "attachment; filename=testfile.fits"} ) dl = Downloader() dl.enqueue_file(httpserver.url, path=Path(tmpdir), max_splits=None) assert dl.queued_downloads == 1 f = dl.download() validate_test_file(f) def test_simple_download(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content( "SIMPLE = T", headers={"Content-Disposition": "attachment; filename=testfile.fits"} ) f = Downloader.simple_download([httpserver.url], path=Path(tmpdir)) validate_test_file(f) def test_changed_max_conn(httpserver, tmpdir): # Check that changing max_conn works after creating Downloader tmpdir = str(tmpdir) httpserver.serve_content( "SIMPLE = T", headers={"Content-Disposition": "attachment; filename=testfile.fits"} ) dl = Downloader(max_conn=4) dl.enqueue_file(httpserver.url, path=Path(tmpdir), max_splits=None) dl.max_conn = 3 f = dl.download() validate_test_file(f) @pytest.mark.asyncio @pytest.mark.parametrize("use_aiofiles", [True, False]) async def test_async_download(httpserver, tmpdir, use_aiofiles): httpserver.serve_content( "SIMPLE = T", headers={"Content-Disposition": "attachment; filename=testfile.fits"} ) dl = Downloader(config=SessionConfig(use_aiofiles=use_aiofiles)) dl.enqueue_file(httpserver.url, path=Path(tmpdir), max_splits=None) assert dl.queued_downloads == 1 f = await dl.run_download() validate_test_file(f) def test_download_ranged_http(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content( "SIMPLE = T", headers={"Content-Disposition": "attachment; filename=testfile.fits"} ) dl = Downloader() dl.enqueue_file(httpserver.url, path=Path(tmpdir)) assert dl.queued_downloads == 1 f = dl.download() validate_test_file(f) def test_regression_download_ranged_http(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content( "S", headers={ "Content-Disposition": "attachment; filename=testfile.fits", "Accept-Ranges": "bytes", }, ) dl = Downloader() dl.enqueue_file(httpserver.url, path=Path(tmpdir)) assert dl.queued_downloads == 1 f = dl.download() assert len(f.errors) == 0, f.errors def test_download_partial(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content("SIMPLE = T") dl = Downloader() dl.enqueue_file(httpserver.url, filename=lambda resp, url: Path(tmpdir) / "filename") f = dl.download() assert len(f) == 1 # strip the http:// assert "filename" in f[0] def test_empty_download(tmpdir): dl = Downloader() f = dl.download() assert len(f) == 0 def test_download_filename(httpserver, tmpdir): httpserver.serve_content("SIMPLE = T") fname = "testing123" filename = str(tmpdir.join(fname)) with open(filename, "w") as fh: fh.write("SIMPLE = T") dl = Downloader() dl.enqueue_file(httpserver.url, filename=filename, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == 1 assert f[0] == filename def test_download_no_overwrite(httpserver, tmpdir): httpserver.serve_content("SIMPLE = T") fname = "testing123" filename = str(tmpdir.join(fname)) with open(filename, "w") as fh: fh.write("Hello world") dl = Downloader() dl.enqueue_file(httpserver.url, filename=filename, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == 1 assert f[0] == filename with open(filename) as fh: # If the contents is the same as when we wrote it, it hasn't been # overwritten assert fh.read() == "Hello world" def test_download_overwrite(httpserver, tmpdir): httpserver.serve_content("SIMPLE = T") fname = "testing123" filename = str(tmpdir.join(fname)) with open(filename, "w") as fh: fh.write("Hello world") dl = Downloader(overwrite=True) dl.enqueue_file(httpserver.url, filename=filename, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == 1 assert f[0] == filename with open(filename) as fh: assert fh.read() == "SIMPLE = T" def test_download_unique(httpserver, tmpdir): httpserver.serve_content("SIMPLE = T") fname = "testing123" filename = str(tmpdir.join(fname)) filenames = [filename, filename + ".fits", filename + ".fits.gz"] dl = Downloader(overwrite="unique") # Write files to both the target filenames. for fn in filenames: with open(fn, "w") as fh: fh.write("Hello world") dl.enqueue_file(httpserver.url, filename=fn, chunksize=200) f = dl.download() assert isinstance(f, Results) assert len(f) == len(filenames) for fn in f: assert fn not in filenames assert f"{fname}.1" in fn def test_retrieve_some_content(testserver, tmpdir): """ Test that the downloader handles errors properly. """ tmpdir = str(tmpdir) dl = Downloader() nn = 5 for i in range(nn): dl.enqueue_file(testserver.url, path=tmpdir) f = dl.download() assert len(f) == nn - 1 assert len(f.errors) == 1 def test_no_progress(httpserver, tmpdir, capsys): tmpdir = str(tmpdir) httpserver.serve_content("SIMPLE = T") dl = Downloader(progress=False) dl.enqueue_file(httpserver.url, path=tmpdir) dl.download() # Check that there was not stdout captured = capsys.readouterr().out assert not captured def throwerror(*args, **kwargs): raise ValueError("Out of Cheese.") @patch("parfive.downloader.default_name", throwerror) def test_raises_other_exception(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content("SIMPLE = T") dl = Downloader() dl.enqueue_file(httpserver.url, path=tmpdir) res = dl.download() assert isinstance(res.errors[0].exception, ValueError) def test_token(): t = Token(5) assert "5" in repr(t) assert "5" in str(t) def test_failed_download(): err = FailedDownload("wibble", "bbc.co.uk", "running away") assert "bbc.co.uk" in repr(err) assert "bbc.co.uk" in repr(err) assert "running away" in str(err) assert "running away" in str(err) def test_results(): res = Results() res.append("hello") res.add_error("wibble", "notaurl", "out of cheese") assert "notaurl" in repr(res) assert "hello" in repr(res) assert "out of cheese" in repr(res) def test_notaurl(tmpdir): tmpdir = str(tmpdir) dl = Downloader(progress=False) dl.enqueue_file("http://notaurl.wibble/file", path=tmpdir) f = dl.download() assert len(f.errors) == 1 assert isinstance(f.errors[0].exception, aiohttp.ClientConnectionError) def test_retry(tmpdir, testserver): tmpdir = str(tmpdir) dl = Downloader() nn = 5 for i in range(nn): dl.enqueue_file(testserver.url, path=tmpdir) f = dl.download() assert len(f) == nn - 1 assert len(f.errors) == 1 f2 = dl.retry(f) assert len(f2) == nn assert len(f2.errors) == 0 def test_empty_retry(): f = Results() dl = Downloader() dl.retry(f) @skip_windows @pytest.mark.allow_hosts(True) def test_ftp(tmpdir): tmpdir = str(tmpdir) dl = Downloader() dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2011_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2013_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://notaserver/notafile.fileL", path=tmpdir) f = dl.download() assert len(f) == 1 assert len(f.errors) == 3 @skip_windows @pytest.mark.allow_hosts(True) def test_ftp_pasv_command(tmpdir): tmpdir = str(tmpdir) dl = Downloader() dl.enqueue_file( "ftp://ftp.ngdc.noaa.gov/STP/swpc_products/daily_reports/solar_region_summaries/2002/04/20020414SRS.txt", path=tmpdir, passive_commands=["pasv"], ) assert dl.queued_downloads == 1 f = dl.download() assert len(f) == 1 assert len(f.errors) == 0 @skip_windows @pytest.mark.allow_hosts(True) def test_ftp_http(tmpdir, httpserver): tmpdir = str(tmpdir) httpserver.serve_content("SIMPLE = T") dl = Downloader() dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2011_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/warehouse/2011/2013_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://ftp.swpc.noaa.gov/pub/_SRS.tar.gz", path=tmpdir) dl.enqueue_file("ftp://notaserver/notafile.fileL", path=tmpdir) dl.enqueue_file(httpserver.url, path=tmpdir) dl.enqueue_file("http://noaurl.notadomain/noafile", path=tmpdir) assert dl.queued_downloads == 6 f = dl.download() assert len(f) == 2 assert len(f.errors) == 4 def test_default_user_agent(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content( "SIMPLE = T", headers={"Content-Disposition": "attachment; filename=testfile.fits"} ) dl = Downloader() dl.enqueue_file(httpserver.url, path=Path(tmpdir), max_splits=None) assert dl.queued_downloads == 1 dl.download() assert "User-Agent" in httpserver.requests[0].headers assert ( httpserver.requests[0].headers["User-Agent"] == f"parfive/{parfive.__version__} aiohttp/{aiohttp.__version__} python/{platform.python_version()}" ) def test_custom_user_agent(httpserver, tmpdir): tmpdir = str(tmpdir) httpserver.serve_content( "SIMPLE = T", headers={"Content-Disposition": "attachment; filename=testfile.fits"} ) dl = Downloader(config=SessionConfig(headers={"User-Agent": "test value 299792458"})) dl.enqueue_file(httpserver.url, path=Path(tmpdir), max_splits=None) assert dl.queued_downloads == 1 dl.download() assert "User-Agent" in httpserver.requests[0].headers assert httpserver.requests[0].headers["User-Agent"] == "test value 299792458" @patch.dict(os.environ, {"HTTP_PROXY": "http_proxy_url", "HTTPS_PROXY": "https_proxy_url"}) @pytest.mark.parametrize( "url,proxy", [ ("http://test.example.com", "http_proxy_url"), ("https://test.example.com", "https_proxy_url"), ], ) def test_proxy_passed_as_kwargs_to_get(tmpdir, url, proxy): with mock.patch("aiohttp.client.ClientSession._request", new_callable=mock.MagicMock) as patched: dl = Downloader() dl.enqueue_file(url, path=Path(tmpdir), max_splits=None) assert dl.queued_downloads == 1 dl.download() assert patched.called, "`ClientSession._request` not called" assert list(patched.call_args) == [ ("GET", url), { "allow_redirects": True, "timeout": ClientTimeout(total=0, connect=None, sock_read=90, sock_connect=None), "proxy": proxy, }, ] ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/tests/test_downloader_multipart.py0000644000175100001710000000237514326506373023234 0ustar00runnerdockerfrom functools import partial from parfive import Downloader from parfive.tests.localserver import error_on_nth_request from parfive.utils import MultiPartDownloadError def test_multipart(multipartserver, tmp_path): dl = Downloader(progress=False) max_splits = 5 dl.enqueue_file(multipartserver.url, path=tmp_path, max_splits=max_splits) files = dl.download() # Verify we transferred all the content with open(files[0], "rb") as fobj: assert fobj.read() == b"a" * 100 # Assert that we made the expected number of requests assert len(multipartserver.requests) == max_splits + 1 assert "HTTP_RANGE" not in multipartserver.requests[0] for split_req in multipartserver.requests[1:]: assert "HTTP_RANGE" in split_req def test_multipart_with_error(multipartserver, tmp_path): multipartserver.callback = partial(error_on_nth_request, 3) dl = Downloader(progress=False) max_splits = 5 dl.enqueue_file(multipartserver.url, path=tmp_path, max_splits=max_splits) files = dl.download() assert len(files) == 0 assert len(files.errors) == 1 assert isinstance(files.errors[0].exception, MultiPartDownloadError) expected_file = tmp_path / "testfile.txt" assert not expected_file.exists() ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/tests/test_main.py0000644000175100001710000000406114326506373017713 0ustar00runnerdockerimport os import pytest from parfive.main import parse_args, run_parfive REQUIRED_ARGUMENTS = ["test_url"] def test_no_url(): with pytest.raises(SystemExit): parse_args(["--overwrite"]) def helper(args, name, expected): args = parse_args(REQUIRED_ARGUMENTS + args) assert getattr(args, name) == expected def test_overwrite(): helper(["--overwrite"], "overwrite", True) helper([], "overwrite", False) def test_max_conn(): helper(["--max-conn", "10"], "max_conn", 10) helper([], "max_conn", 5) def test_max_splits(): helper(["--max-splits", "10"], "max_splits", 10) helper([], "max_splits", 5) def test_no_file_progress(): helper(["--no-file-progress"], "no_file_progress", True) helper([], "no_file_progress", False) def test_no_progress(): helper(["--no-progress"], "no_progress", True) helper([], "no_progress", False) def test_print_filenames(): helper(["--print-filenames"], "print_filenames", True) helper([], "print_filenames", False) def test_directory(): helper(["--directory", "/tmp"], "directory", "/tmp") helper([], "directory", "./") def test_verbose(): helper(["--verbose"], "verbose", True) helper([], "verbose", False) @pytest.fixture def test_url(multipartserver): return multipartserver.url @pytest.mark.parametrize( "args", [ [], ["--no-progress"], ["--print-filenames"], ["--verbose"], ], ) def test_run_cli_success(args, test_url, capsys): cliargs = parse_args(args + [test_url]) with pytest.raises(SystemExit) as exit_exc: run_parfive(cliargs) assert exit_exc.value.code == 0 cap_out = capsys.readouterr() if "--print-filenames" in args: assert "testfile.txt" in cap_out.out else: assert "testfile.txt" not in cap_out.out if "--no-progress" in args: assert "Files Downloaded:" not in cap_out.err else: assert "Files Downloaded:" in cap_out.err if "--verbose" in args: assert "DEBUG" in cap_out.err os.remove("testfile.txt") ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/tests/test_utils.py0000644000175100001710000000045714326506373020134 0ustar00runnerdockerimport tempfile from parfive.utils import sha256sum def test_sha256sum(): tempfilename = tempfile.mktemp() filehash = "559aead08264d5795d3909718cdd05abd49572e84fe55590eef31a88a08fdffd" with open(tempfilename, "w") as f: f.write("A") assert sha256sum(tempfilename) == filehash ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/parfive/utils.py0000644000175100001710000001404014326506373015724 0ustar00runnerdockerimport os import asyncio import hashlib import pathlib import warnings from pathlib import Path from itertools import count from concurrent.futures import ThreadPoolExecutor import aiohttp import parfive __all__ = [ "cancel_task", "run_in_thread", "Token", "FailedDownload", "default_name", "remove_file", ] # Copied out of CPython under PSF Licence 2 def _parseparam(s): while s[:1] == ";": s = s[1:] end = s.find(";") while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: end = s.find(";", end + 1) if end < 0: end = len(s) f = s[:end] yield f.strip() s = s[end:] def parse_header(line): """Parse a Content-type like header. Return the main content-type and a dictionary of options. """ parts = _parseparam(";" + line) key = parts.__next__() pdict = {} for p in parts: i = p.find("=") if i >= 0: name = p[:i].strip().lower() value = p[i + 1 :].strip() if len(value) >= 2 and value[0] == value[-1] == '"': value = value[1:-1] value = value.replace("\\\\", "\\").replace('\\"', '"') pdict[name] = value return key, pdict def default_name(path: os.PathLike, resp: aiohttp.ClientResponse, url: str) -> os.PathLike: url_filename = url.split("/")[-1] if resp: cdheader = resp.headers.get("Content-Disposition", None) if cdheader: value, params = parse_header(cdheader) name = params.get("filename", url_filename) else: name = url_filename else: name = url_filename return pathlib.Path(path) / name def run_task_in_thread(loop, coro): """ This function returns the asyncio Future after running the loop in a thread. This makes the return value of this function the same as the return of ``loop.run_until_complete``. """ with ThreadPoolExecutor(max_workers=1) as aio_pool: try: future = aio_pool.submit(loop.run_until_complete, coro) except KeyboardInterrupt: future.cancel() return future.result() async def get_ftp_size(client, filepath): """ Given an `aioftp.ClientSession` object get the expected size of the file, return ``None`` if the size can not be determined. """ try: size = await client.stat(filepath) size = size.get("size", None) except Exception: parfive.log.exception("Failed to get size of FTP file") size = None return int(size) if size else size def get_http_size(resp): size = resp.headers.get("content-length", None) return int(size) if size else size def replacement_filename(path): """ Given a path generate a unique filename. """ path = pathlib.Path(path) if not path.exists: return path suffix = "".join(path.suffixes) for c in count(1): if suffix: name, _ = path.name.split(suffix) else: name = path.name new_name = f"{name}.{c}{suffix}" new_path = path.parent / new_name if not new_path.exists(): return new_path def get_filepath(filepath, overwrite): """ Get the filepath to download to and ensure dir exists. Returns ------- `pathlib.Path`, `bool` """ filepath = pathlib.Path(filepath) if filepath.exists(): if not overwrite: return str(filepath), True if overwrite == "unique": filepath = replacement_filename(filepath) if not filepath.parent.exists(): filepath.parent.mkdir(parents=True) return filepath, False def sha256sum(filename): """ https://stackoverflow.com/a/44873382 """ h = hashlib.sha256() b = bytearray(128 * 1024) mv = memoryview(b) with open(filename, "rb", buffering=0) as f: for n in iter(lambda: f.readinto(mv), 0): h.update(mv[:n]) return h.hexdigest() class MultiPartDownloadError(Exception): def __init__(self, response): self.response = response class FailedDownload(Exception): def __init__(self, filepath_partial, url, exception): self.filepath_partial = filepath_partial self.url = url self.exception = exception super().__init__() def __repr__(self): out = super().__repr__() out += f"\n {self.url} {self.exception}" return out def __str__(self): return "Download Failed: {} with error {}".format(self.url, str(self.exception)) class Token: def __init__(self, n): self.n = n def __repr__(self): return super().__repr__() + f"n = {self.n}" def __str__(self): return f"Token {self.n}" class _QueueList(list): """ A list, with an extra method that empties the list and puts it into a `asyncio.Queue`. Creating the queue can only be done inside a running asyncio loop. """ def generate_queue(self, maxsize=0): queue = asyncio.Queue(maxsize=maxsize) for item in self: queue.put_nowait(item) self.clear() return queue class ParfiveUserWarning(UserWarning): """ Raised for not-quite errors. """ class ParfiveFutureWarning(FutureWarning): """ Raised for future changes to the parfive API. """ def remove_file(filepath): """ Remove the file from the disk, if it exists """ filepath = Path(filepath) try: # When we drop 3.7 support we can use unlink(missing_ok=True) if filepath.exists(): filepath.unlink() except Exception as remove_exception: warnings.warn( f"Failed to delete possibly incomplete file {filepath} {remove_exception}", ParfiveUserWarning, ) async def cancel_task(task): """ Call cancel on a task and then wait for it to exit. Return True if the task was cancelled, False otherwise. """ task.cancel() try: await task except asyncio.CancelledError: return True return task.cancelled() ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4248548 parfive-2.0.2/parfive.egg-info/0000755000175100001710000000000014326506417015704 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878735.0 parfive-2.0.2/parfive.egg-info/PKG-INFO0000644000175100001710000000700214326506417017000 0ustar00runnerdockerMetadata-Version: 2.1 Name: parfive Version: 2.0.2 Summary: A HTTP and FTP parallel file downloader. Home-page: https://parfive.readthedocs.io/ Author: "Stuart Mumford" Author-email: "stuart@cadair.com" License: MIT Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Requires-Python: >=3.7 Provides-Extra: ftp Provides-Extra: tests Provides-Extra: docs License-File: LICENSE Parfive ======= .. image:: https://img.shields.io/pypi/v/parfive.svg :target: https://pypi.python.org/pypi/parfive :alt: Latest PyPI version A parallel file downloader using asyncio. parfive can handle downloading multiple files in parallel as well as downloading each file in a number of chunks. Usage ----- .. image:: https://asciinema.org/a/EuALahgkiicWHGmrfFsZSLz81.svg :alt: asciicast demo of parfive :target: https://asciinema.org/a/EuALahgkiicWHGmrfFsZSLz81 parfive works by creating a downloader object, appending files to it and then running the download. parfive has a synchronous API, but uses asyncio to paralellise downloading the files. A simple example is:: from parfive import Downloader dl = Downloader() dl.enqueue_file("http://data.sunpy.org/sample-data/predicted-sunspot-radio-flux.txt", path="./") files = dl.download() Parfive also bundles a CLI. The following example will download the two files concurrently.:: $ parfive 'http://212.183.159.230/5MB.zip' 'http://212.183.159.230/10MB.zip' $ parfive --help usage: parfive [-h] [--max-conn MAX_CONN] [--overwrite] [--no-file-progress] [--directory DIRECTORY] [--print-filenames] URLS [URLS ...] Parfive, the python asyncio based downloader positional arguments: URLS URLs of files to be downloaded. optional arguments: -h, --help show this help message and exit --max-conn MAX_CONN Number of maximum connections. --overwrite Overwrite if the file exists. --no-file-progress Show progress bar for each file. --directory DIRECTORY Directory to which downloaded files are saved. --print-filenames Print successfully downloaded files's names to stdout. Results ^^^^^^^ ``parfive.Downloader.download`` returns a ``parfive.Results`` object, which is a list of the filenames that have been downloaded. It also tracks any files which failed to download. Handling Errors ^^^^^^^^^^^^^^^ If files fail to download, the urls and the response from the server are stored in the ``Results`` object returned by ``parfive.Downloader``. These can be used to inform users about the errors. (Note, the progress bar will finish in an incomplete state if a download fails, i.e. it will show ``4/5 Files Downloaded``). The ``Results`` object is a list with an extra attribute ``errors``, this property returns a list of named tuples, where these named tuples contains the ``.url`` and the ``.response``, which is a ``aiohttp.ClientResponse`` or a ``aiohttp.ClientError`` object. Installation ------------ parfive is available on PyPI, you can install it with pip:: pip install parfive or if you want to use FTP downloads:: pip install parfive[ftp] Requirements ^^^^^^^^^^^^ - Python 3.7 or above - aiohttp - tqdm - aioftp (for downloads over FTP) Licence ------- MIT Licensed Authors ------- `parfive` was written by `Stuart Mumford `__. ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878735.0 parfive-2.0.2/parfive.egg-info/SOURCES.txt0000644000175100001710000000170514326506417017573 0ustar00runnerdocker.codecov.yaml .gitignore .pre-commit-config.yaml .readthedocs.yml LICENSE README.rst pyproject.toml setup.cfg setup.py tox.ini .github/FUNDING.yml .github/release-drafter.yml .github/workflows/ci_workflows.yml .github/workflows/release-drafter.yml docs/Makefile docs/conf.py docs/index.rst docs/robots.txt docs/static/css/contributors.css parfive/__init__.py parfive/_version.py parfive/config.py parfive/conftest.py parfive/downloader.py parfive/main.py parfive/results.py parfive/utils.py parfive.egg-info/PKG-INFO parfive.egg-info/SOURCES.txt parfive.egg-info/dependency_links.txt parfive.egg-info/entry_points.txt parfive.egg-info/requires.txt parfive.egg-info/top_level.txt parfive/tests/__init__.py parfive/tests/localserver.py parfive/tests/simple_download_test.ipynb parfive/tests/test_aiofiles.py parfive/tests/test_config.py parfive/tests/test_downloader.py parfive/tests/test_downloader_multipart.py parfive/tests/test_main.py parfive/tests/test_utils.py././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878735.0 parfive-2.0.2/parfive.egg-info/dependency_links.txt0000644000175100001710000000000114326506417021752 0ustar00runnerdocker ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878735.0 parfive-2.0.2/parfive.egg-info/entry_points.txt0000644000175100001710000000005614326506417021203 0ustar00runnerdocker[console_scripts] parfive = parfive.main:main ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878735.0 parfive-2.0.2/parfive.egg-info/requires.txt0000644000175100001710000000041614326506417020305 0ustar00runnerdockertqdm>=4.27.0 aiohttp [:python_version < "3.8"] typing_extensions [docs] sphinx<5 sphinx-automodapi sphinx-autodoc-typehints sphinx-contributors sphinx-book-theme [ftp] aioftp>=0.17.1 [tests] pytest pytest-localserver pytest-asyncio pytest-socket pytest-cov aiofiles ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878735.0 parfive-2.0.2/parfive.egg-info/top_level.txt0000644000175100001710000000001014326506417020425 0ustar00runnerdockerparfive ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/pyproject.toml0000644000175100001710000000021414326506373015470 0ustar00runnerdocker[build-system] requires = ["setuptools", "setuptools_scm", "wheel"] build-backend = 'setuptools.build_meta' [tool.black] line-length = 101 ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1666878735.4248548 parfive-2.0.2/setup.cfg0000644000175100001710000000423314326506417014401 0ustar00runnerdocker[metadata] name = parfive description = A HTTP and FTP parallel file downloader. long_description = file: README.rst url = https://parfive.readthedocs.io/ license = MIT author = "Stuart Mumford" author_email = "stuart@cadair.com" classifiers = Programming Language :: Python :: 3 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 [options] python_requires = >=3.7 install_requires = tqdm >= 4.27.0 aiohttp typing_extensions;python_version<'3.8' setup_requires = setuptools_scm packages = find: [options.entry_points] console_scripts = parfive = parfive.main:main [options.extras_require] ftp = aioftp>=0.17.1 tests = pytest pytest-localserver pytest-asyncio pytest-socket pytest-cov aiofiles docs = sphinx<5 sphinx-automodapi sphinx-autodoc-typehints sphinx-contributors sphinx-book-theme [flake8] max-line-length = 100 ignore = I100,I101,I102,I103,I104,I201 [tool:pytest] addopts = --allow-hosts=127.0.0.1,::1 asyncio_mode = strict [isort] profile = black balanced_wrapping = True skip = docs/conf.py default_section = THIRDPARTY include_trailing_comma = True known_first_party = parfive length_sort = False length_sort_sections = stdlib line_length = 110 multi_line_output = 3 no_lines_before = LOCALFOLDER sections = STDLIB, THIRDPARTY, FIRSTPARTY, LOCALFOLDER [coverage:run] omit = parfive/__init__* parfive/_dev/* parfive/*setup* parfive/conftest.py parfive/tests/* parfive/version.py */parfive/__init__* */parfive/_dev/* */parfive/*setup* */parfive/conftest.py */parfive/tests/* */parfive/version.py [coverage:report] exclude_lines = pragma: no cover except ImportError raise AssertionError raise NotImplementedError def main\(.*\): pragma: py{ignore_python_version} [mypy] plugins = pydantic.mypy [mypy-parfive._version] ignore_missing_imports = True [mypy-tqdm.*] ignore_missing_imports = True [mypy-pytest_localserver.*] ignore_missing_imports = True [mypy-aioftp.*] ignore_missing_imports = True [mypy-aiohttp.*] ignore_missing_imports = True [mypy-pytest.*] ignore_missing_imports = True [egg_info] tag_build = tag_date = 0 ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/setup.py0000644000175100001710000000023714326506373014273 0ustar00runnerdocker#!/usr/bin/env python from setuptools import setup # isort:skip import os setup( use_scm_version={"write_to": os.path.join("parfive", "_version.py")}, ) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1666878715.0 parfive-2.0.2/tox.ini0000644000175100001710000000266214326506373014100 0ustar00runnerdocker[tox] envlist = py{37,38,39,310}{,-conda} codestyle build_docs isolated_build = True [testenv] setenv = extras = ftp tests commands = pytest -vvv -s -raR --pyargs parfive --cov-report=xml --cov=parfive --cov-config={toxinidir}/setup.cfg {toxinidir}/docs {posargs} [testenv:build_docs] changedir = docs description = Invoke sphinx-build to build the HTML docs # Be verbose about the extras rather than using dev for clarity extras = ftp docs commands = sphinx-build -j auto --color -W --keep-going -b html -d _build/.doctrees . _build/html {posargs} python -c 'import pathlib; print("Documentation available under file://\{0\}".format(pathlib.Path(r"{toxinidir}") / "docs" / "_build" / "index.html"))' [testenv:codestyle] skip_install = true description = Run all style and file checks with pre-commit deps = pre-commit commands = pre-commit install-hooks pre-commit run --color always --all-files --show-diff-on-failure [testenv:mypy] skip_install = true description = Run mypy deps = mypy types-aiofiles pydantic commands = mypy -p parfive # This env requires tox-conda. [testenv:conda] extras = deps = conda_deps = aioftp aiohttp pytest-asyncio pytest-cov pytest-localserver pytest-socket pytest-sugar tqdm conda_channels = conda-forge install_command = pip install --no-deps {opts} {packages} commands = conda list {env:PYTEST_COMMAND} {posargs}