pax_global_header00006660000000000000000000000064144646552500014525gustar00rootroot0000000000000052 comment=942a41084dbe79743d3114a475dc9e390b8718f1 joblib-1.3.2/000077500000000000000000000000001446465525000127715ustar00rootroot00000000000000joblib-1.3.2/.codecov.yml000066400000000000000000000003321446465525000152120ustar00rootroot00000000000000codecov: token: 1b7eb264-fd77-469a-829a-e9cd5efd7cef coverage: status: project: default: # Allow coverage to drop by up to 1% in a PR before marking it as # failed threshold: '1%' joblib-1.3.2/.gitignore000066400000000000000000000012701446465525000147610ustar00rootroot00000000000000*.py[oc] *.so # setup.py working directory build # setup.py dist directory dist # Editor temporary/working/backup files *$ .*.sw[nop] .sw[nop] *~ [#]*# .#* *.bak *.tmp *.tgz *.rej *.org .project *.diff .settings/ *.svn/ # Egg metadata *.egg-info # The shelf plugin uses this dir ./.shelf # Some IDEs add this directory .idea .vscode # Mac droppings .DS_Store doc/documentation.zip doc/generated doc/CHANGES.rst doc/README.rst # Coverage report .coverage # pytest cache on failure .cache .pytest_cache # Generated ctags tags # Generated by sphinx doc/auto_examples/ doc/_build # Dask worker space directory in generated examples dask-worker-space/ # Binary files created by benchmarks *.npy joblib-1.3.2/.mailmap000066400000000000000000000007231446465525000144140ustar00rootroot00000000000000Gael Varoquaux Gael Varoquaux Gael Varoquaux Gael varoquaux Gael Varoquaux GaelVaroquaux Gael Varoquaux Gael VAROQUAUX Gael Varoquaux gvaroquaux joblib-1.3.2/.readthedocs-requirements.txt000066400000000000000000000001431446465525000206140ustar00rootroot00000000000000sphinx docutils<0.18 numpy matplotlib pillow sphinx-gallery numpydoc pandas lz4 distributed psutil joblib-1.3.2/.readthedocs.yaml000066400000000000000000000003471446465525000162240ustar00rootroot00000000000000version: 2 python: # make sure joblib is installed in the virtualenv (it is imported in # conf.py) install: - requirements: .readthedocs-requirements.txt - method: pip path: . sphinx: fail_on_warning: true joblib-1.3.2/CHANGES.rst000066400000000000000000001134721446465525000146030ustar00rootroot00000000000000Latest changes ============== Release 1.3.2 -- 2023/08/08 --------------------------- - Fix a regression in ``joblib.Parallel`` introduced in 1.3.0 where explicitly setting ``n_jobs=None`` was not interpreted as "unset". https://github.com/joblib/joblib/pull/1475 - Fix a regression in ``joblib.Parallel`` introduced in 1.3.0 where ``joblib.Parallel`` logging methods exposed from inheritance to ``joblib.Logger`` didn't work because of missing logger initialization. https://github.com/joblib/joblib/pull/1494 - Various maintenance updates to the doc, the ci and the test. https://github.com/joblib/joblib/pull/1480, https://github.com/joblib/joblib/pull/1481, https://github.com/joblib/joblib/pull/1476, https://github.com/joblib/joblib/pull/1492 Release 1.3.1 -- 2023/06/29 --------------------------- - Fix compatibility with python 3.7 by vendor loky 3.4.1 which is compatible with this version. https://github.com/joblib/joblib/pull/1472 Release 1.3.0 -- 2023/06/28 --------------------------- - Ensure native byte order for memmap arrays in ``joblib.load``. https://github.com/joblib/joblib/issues/1353 - Add ability to change default Parallel backend in tests by setting the ``JOBLIB_TESTS_DEFAULT_PARALLEL_BACKEND`` environment variable. https://github.com/joblib/joblib/pull/1356 - Fix temporary folder creation in `joblib.Parallel` on Linux subsystems on Windows which do have `/dev/shm` but don't have the `os.statvfs` function https://github.com/joblib/joblib/issues/1353 - Drop runtime dependency on ``distutils``. ``distutils`` is going away in Python 3.12 and is deprecated from Python 3.10 onwards. This import was kept around to avoid breaking scikit-learn, however it's now been long enough since scikit-learn deployed a fixed (verion 1.1 was released in May 2022) that it should be safe to remove this. https://github.com/joblib/joblib/pull/1361 - A warning is raised when a pickling error occurs during caching operations. In version 1.5, this warning will be turned into an error. For all other errors, a new warning has been introduced: ``joblib.memory.CacheWarning``. https://github.com/joblib/joblib/pull/1359 - Avoid (module, name) collisions when caching nested functions. This fix changes the module name of nested functions, invalidating caches from previous versions of Joblib. https://github.com/joblib/joblib/pull/1374 - Add ``cache_validation_callback`` in :meth:`joblib.Memory.cache`, to allow custom cache invalidation based on the metadata of the function call. https://github.com/joblib/joblib/pull/1149 - Add a ``return_as`` parameter for ``Parallel``, that enables consuming results asynchronously. https://github.com/joblib/joblib/pull/1393, https://github.com/joblib/joblib/pull/1458 - Improve the behavior of ``joblib`` for ``n_jobs=1``, with simplified tracebacks and more efficient running time. https://github.com/joblib/joblib/pull/1393 - Add the ``parallel_config`` context manager to allow for more fine-grained control over the backend configuration. It should be used in place of the ``parallel_backend`` context manager. In particular, it has the advantage of not requiring to set a specific backend in the context manager. https://github.com/joblib/joblib/pull/1392, https://github.com/joblib/joblib/pull/1457 - Add ``items_limit`` and ``age_limit`` in :meth:`joblib.Memory.reduce_size` to make it easy to limit the number of items and remove items that have not been accessed for a long time in the cache. https://github.com/joblib/joblib/pull/1200 - Deprecate ``bytes_limit`` in ``Memory`` as this is not automatically enforced, the limit can be directly passed to :meth:`joblib.Memory.reduce_size` which needs to be called to actually enforce the limit. https://github.com/joblib/joblib/pull/1447 - Vendor ``loky`` 3.4.0 which includes various fixes. https://github.com/joblib/joblib/pull/1422 - Various updates to the documentation and to benchmarking tools. https://github.com/joblib/joblib/pull/1343, https://github.com/joblib/joblib/pull/1348, https://github.com/joblib/joblib/pull/1411, https://github.com/joblib/joblib/pull/1451, https://github.com/joblib/joblib/pull/1427, https://github.com/joblib/joblib/pull/1400 - Move project metadata to ``pyproject.toml``. https://github.com/joblib/joblib/pull/1382, https://github.com/joblib/joblib/pull/1433 - Add more tests to improve python ``nogil`` support. https://github.com/joblib/joblib/pull/1394, https://github.com/joblib/joblib/pull/1395 Release 1.2.0 ------------- - Fix a security issue where ``eval(pre_dispatch)`` could potentially run arbitrary code. Now only basic numerics are supported. https://github.com/joblib/joblib/pull/1327 - Make sure that joblib works even when multiprocessing is not available, for instance with Pyodide https://github.com/joblib/joblib/pull/1256 - Avoid unnecessary warnings when workers and main process delete the temporary memmap folder contents concurrently. https://github.com/joblib/joblib/pull/1263 - Fix memory alignment bug for pickles containing numpy arrays. This is especially important when loading the pickle with ``mmap_mode != None`` as the resulting ``numpy.memmap`` object would not be able to correct the misalignment without performing a memory copy. This bug would cause invalid computation and segmentation faults with native code that would directly access the underlying data buffer of a numpy array, for instance C/C++/Cython code compiled with older GCC versions or some old OpenBLAS written in platform specific assembly. https://github.com/joblib/joblib/pull/1254 - Vendor cloudpickle 2.2.0 which adds support for PyPy 3.8+. - Vendor loky 3.3.0 which fixes several bugs including: - robustly forcibly terminating worker processes in case of a crash (https://github.com/joblib/joblib/pull/1269); - avoiding leaking worker processes in case of nested loky parallel calls; - reliability spawn the correct number of reusable workers. Release 1.1.1 ------------- - Fix a security issue where ``eval(pre_dispatch)`` could potentially run arbitrary code. Now only basic numerics are supported. https://github.com/joblib/joblib/pull/1327 Release 1.1.0 -------------- - Fix byte order inconsistency issue during deserialization using joblib.load in cross-endian environment: the numpy arrays are now always loaded to use the system byte order, independently of the byte order of the system that serialized the pickle. https://github.com/joblib/joblib/pull/1181 - Fix joblib.Memory bug with the ``ignore`` parameter when the cached function is a decorated function. https://github.com/joblib/joblib/pull/1165 - Fix `joblib.Memory` to properly handle caching for functions defined interactively in a IPython session or in Jupyter notebook cell. https://github.com/joblib/joblib/pull/1214 - Update vendored loky (from version 2.9 to 3.0) and cloudpickle (from version 1.6 to 2.0) https://github.com/joblib/joblib/pull/1218 Release 1.0.1 ------------- - Add check_call_in_cache method to check cache without calling function. https://github.com/joblib/joblib/pull/820 - dask: avoid redundant scattering of large arguments to make a more efficient use of the network resources and avoid crashing dask with "OSError: [Errno 55] No buffer space available" or "ConnectionResetError: [Errno 104] connection reset by peer". https://github.com/joblib/joblib/pull/1133 Release 1.0.0 ------------- - Make `joblib.hash` and `joblib.Memory` caching system compatible with `numpy >= 1.20.0`. Also make it explicit in the documentation that users should now expect to have their `joblib.Memory` cache invalidated when either `joblib` or a third party library involved in the cached values definition is upgraded. In particular, users updating `joblib` to a release that includes this fix will see their previous cache invalidated if they contained reference to `numpy` objects. https://github.com/joblib/joblib/pull/1136 - Remove deprecated `check_pickle` argument in `delayed`. https://github.com/joblib/joblib/pull/903 Release 0.17.0 -------------- - Fix a spurious invalidation of `Memory.cache`'d functions called with `Parallel` under Jupyter or IPython. https://github.com/joblib/joblib/pull/1093 - Bump vendored loky to 2.9.0 and cloudpickle to 1.6.0. In particular this fixes a problem to add compat for Python 3.9. Release 0.16.0 -------------- - Fix a problem in the constructors of Parallel backends classes that inherit from the `AutoBatchingMixin` that prevented the dask backend to properly batch short tasks. https://github.com/joblib/joblib/pull/1062 - Fix a problem in the way the joblib dask backend batches calls that would badly interact with the dask callable pickling cache and lead to wrong results or errors. https://github.com/joblib/joblib/pull/1055 - Prevent a dask.distributed bug from surfacing in joblib's dask backend during nested Parallel calls (due to joblib's auto-scattering feature) https://github.com/joblib/joblib/pull/1061 - Workaround for a race condition after Parallel calls with the dask backend that would cause low level warnings from asyncio coroutines: https://github.com/joblib/joblib/pull/1078 Release 0.15.1 -------------- - Make joblib work on Python 3 installation that do not ship with the lzma package in their standard library. Release 0.15.0 -------------- - Drop support for Python 2 and Python 3.5. All objects in ``joblib.my_exceptions`` and ``joblib.format_stack`` are now deprecated and will be removed in joblib 0.16. Note that no deprecation warning will be raised for these objects Python < 3.7. https://github.com/joblib/joblib/pull/1018 - Fix many bugs related to the temporary files and folder generated when automatically memory mapping large numpy arrays for efficient inter-process communication. In particular, this would cause `PermissionError` exceptions to be raised under Windows and large leaked files in `/dev/shm` under Linux in case of crash. https://github.com/joblib/joblib/pull/966 - Make the dask backend collect results as soon as they complete leading to a performance improvement: https://github.com/joblib/joblib/pull/1025 - Fix the number of jobs reported by ``effective_n_jobs`` when ``n_jobs=None`` called in a parallel backend context. https://github.com/joblib/joblib/pull/985 - Upgraded vendored cloupickle to 1.4.1 and loky to 2.8.0. This allows for Parallel calls of dynamically defined functions with type annotations in particular. Release 0.14.1 -------------- - Configure the loky workers' environment to mitigate oversubsription with nested multi-threaded code in the following case: - allow for a suitable number of threads for numba (``NUMBA_NUM_THREADS``); - enable Interprocess Communication for scheduler coordination when the nested code uses Threading Building Blocks (TBB) (``ENABLE_IPC=1``) https://github.com/joblib/joblib/pull/951 - Fix a regression where the loky backend was not reusing previously spawned workers. https://github.com/joblib/joblib/pull/968 - Revert https://github.com/joblib/joblib/pull/847 to avoid using `pkg_resources` that introduced a performance regression under Windows: https://github.com/joblib/joblib/issues/965 Release 0.14.0 -------------- - Improved the load balancing between workers to avoid stranglers caused by an excessively large batch size when the task duration is varying significantly (because of the combined use of ``joblib.Parallel`` and ``joblib.Memory`` with a partially warmed cache for instance). https://github.com/joblib/joblib/pull/899 - Add official support for Python 3.8: fixed protocol number in `Hasher` and updated tests. - Fix a deadlock when using the dask backend (when scattering large numpy arrays). https://github.com/joblib/joblib/pull/914 - Warn users that they should never use `joblib.load` with files from untrusted sources. Fix security related API change introduced in numpy 1.6.3 that would prevent using joblib with recent numpy versions. https://github.com/joblib/joblib/pull/879 - Upgrade to cloudpickle 1.1.1 that add supports for the upcoming Python 3.8 release among other things. https://github.com/joblib/joblib/pull/878 - Fix semaphore availability checker to avoid spawning resource trackers on module import. https://github.com/joblib/joblib/pull/893 - Fix the oversubscription protection to only protect against nested `Parallel` calls. This allows `joblib` to be run in background threads. https://github.com/joblib/joblib/pull/934 - Fix `ValueError` (negative dimensions) when pickling large numpy arrays on Windows. https://github.com/joblib/joblib/pull/920 - Upgrade to loky 2.6.0 that add supports for the setting environment variables in child before loading any module. https://github.com/joblib/joblib/pull/940 - Fix the oversubscription protection for native libraries using threadpools (OpenBLAS, MKL, Blis and OpenMP runtimes). The maximal number of threads is can now be set in children using the ``inner_max_num_threads`` in ``parallel_backend``. It defaults to ``cpu_count() // n_jobs``. https://github.com/joblib/joblib/pull/940 Release 0.13.2 -------------- Pierre Glaser Upgrade to cloudpickle 0.8.0 Add a non-regression test related to joblib issues #836 and #833, reporting that cloudpickle versions between 0.5.4 and 0.7 introduced a bug where global variables changes in a parent process between two calls to joblib.Parallel would not be propagated into the workers Release 0.13.1 -------------- Pierre Glaser Memory now accepts pathlib.Path objects as ``location`` parameter. Also, a warning is raised if the returned backend is None while ``location`` is not None. Olivier Grisel Make ``Parallel`` raise an informative ``RuntimeError`` when the active parallel backend has zero worker. Make the ``DaskDistributedBackend`` wait for workers before trying to schedule work. This is useful in particular when the workers are provisionned dynamically but provisionning is not immediate (for instance using Kubernetes, Yarn or an HPC job queue). Release 0.13.0 -------------- Thomas Moreau Include loky 2.4.2 with default serialization with ``cloudpickle``. This can be tweaked with the environment variable ``LOKY_PICKLER``. Thomas Moreau Fix nested backend in SequentialBackend to avoid changing the default backend to Sequential. (#792) Thomas Moreau, Olivier Grisel Fix nested_backend behavior to avoid setting the default number of workers to -1 when the backend is not dask. (#784) Release 0.12.5 -------------- Thomas Moreau, Olivier Grisel Include loky 2.3.1 with better error reporting when a worker is abruptly terminated. Also fixes spurious debug output. Pierre Glaser Include cloudpickle 0.5.6. Fix a bug with the handling of global variables by locally defined functions. Release 0.12.4 -------------- Thomas Moreau, Pierre Glaser, Olivier Grisel Include loky 2.3.0 with many bugfixes, notably w.r.t. when setting non-default multiprocessing contexts. Also include improvement on memory management of long running worker processes and fixed issues when using the loky backend under PyPy. Maxime Weyl Raises a more explicit exception when a corrupted MemorizedResult is loaded. Maxime Weyl Loading a corrupted cached file with mmap mode enabled would recompute the results and return them without memory mapping. Release 0.12.3 -------------- Thomas Moreau Fix joblib import setting the global start_method for multiprocessing. Alexandre Abadie Fix MemorizedResult not picklable (#747). Loïc Estève Fix Memory, MemorizedFunc and MemorizedResult round-trip pickling + unpickling (#746). James Collins Fixed a regression in Memory when positional arguments are called as kwargs several times with different values (#751). Thomas Moreau and Olivier Grisel Integration of loky 2.2.2 that fixes issues with the selection of the default start method and improve the reporting when calling functions with arguments that raise an exception when unpickling. Maxime Weyl Prevent MemorizedFunc.call_and_shelve from loading cached results to RAM when not necessary. Results in big performance improvements Release 0.12.2 -------------- Olivier Grisel Integrate loky 2.2.0 to fix regression with unpicklable arguments and functions reported by users (#723, #643). Loky 2.2.0 also provides a protection against memory leaks long running applications when psutil is installed (reported as #721). Joblib now includes the code for the dask backend which has been updated to properly handle nested parallelism and data scattering at the same time (#722). Alexandre Abadie and Olivier Grisel Restored some private API attribute and arguments (`MemorizedResult.argument_hash` and `BatchedCalls.__init__`'s `pickle_cache`) for backward compat. (#716, #732). Joris Van den Bossche Fix a deprecation warning message (for `Memory`'s `cachedir`) (#720). Release 0.12.1 -------------- Thomas Moreau Make sure that any exception triggered when serializing jobs in the queue will be wrapped as a PicklingError as in past versions of joblib. Noam Hershtig Fix kwonlydefaults key error in filter_args (#715) Release 0.12 ------------ Thomas Moreau Implement the ``'loky'`` backend with @ogrisel. This backend relies on a robust implementation of ``concurrent.futures.ProcessPoolExecutor`` with spawned processes that can be reused across the ``Parallel`` calls. This fixes the bad integration with third paty libraries relying on thread pools, described in https://pythonhosted.org/joblib/parallel.html#bad-interaction-of-multiprocessing-and-third-party-libraries Limit the number of threads used in worker processes by C-libraries that relies on threadpools. This functionality works for MKL, OpenBLAS, OpenMP and Accelerated. Elizabeth Sander Prevent numpy arrays with the same shape and data from hashing to the same memmap, to prevent jobs with preallocated arrays from writing over each other. Olivier Grisel Reduce overhead of automatic memmap by removing the need to hash the array. Make ``Memory.cache`` robust to ``PermissionError (errno 13)`` under Windows when run in combination with ``Parallel``. The automatic array memory mapping feature of ``Parallel`` does no longer use ``/dev/shm`` if it is too small (less than 2 GB). In particular in docker containers ``/dev/shm`` is only 64 MB by default which would cause frequent failures when running joblib in Docker containers. Make it possible to hint for thread-based parallelism with ``prefer='threads'`` or enforce shared-memory semantics with ``require='sharedmem'``. Rely on the built-in exception nesting system of Python 3 to preserve traceback information when an exception is raised on a remote worker process. This avoid verbose and redundant exception reports under Python 3. Preserve exception type information when doing nested Parallel calls instead of mapping the exception to the generic ``JoblibException`` type. Alexandre Abadie Introduce the concept of 'store' and refactor the ``Memory`` internal storage implementation to make it accept extra store backends for caching results. ``backend`` and ``backend_options`` are the new options added to ``Memory`` to specify and configure a store backend. Add the ``register_store_backend`` function to extend the store backend used by default with Memory. This default store backend is named 'local' and corresponds to the local filesystem. The store backend API is experimental and thus is subject to change in the future without deprecation. The ``cachedir`` parameter of ``Memory`` is now marked as deprecated, use ``location`` instead. Add support for LZ4 compression if ``lz4`` package is installed. Add ``register_compressor`` function for extending available compressors. Allow passing a string to ``compress`` parameter in ``dump`` function. This string should correspond to the compressor used (e.g. zlib, gzip, lz4, etc). The default compression level is used in this case. Matthew Rocklin Allow ``parallel_backend`` to be used globally instead of only as a context manager. Support lazy registration of external parallel backends Release 0.11 ------------ Alexandre Abadie Remove support for python 2.6 Alexandre Abadie Remove deprecated `format_signature`, `format_call` and `load_output` functions from Memory API. Loïc Estève Add initial implementation of LRU cache cleaning. You can specify the size limit of a ``Memory`` object via the ``bytes_limit`` parameter and then need to clean explicitly the cache via the ``Memory.reduce_size`` method. Olivier Grisel Make the multiprocessing backend work even when the name of the main thread is not the Python default. Thanks to Roman Yurchak for the suggestion. Karan Desai pytest is used to run the tests instead of nosetests. ``python setup.py test`` or ``python setup.py nosetests`` do not work anymore, run ``pytest joblib`` instead. Loïc Estève An instance of ``joblib.ParallelBackendBase`` can be passed into the ``parallel`` argument in ``joblib.Parallel``. Loïc Estève Fix handling of memmap objects with offsets greater than mmap.ALLOCATIONGRANULARITY in ``joblib.Parallel``. See https://github.com/joblib/joblib/issues/451 for more details. Loïc Estève Fix performance regression in ``joblib.Parallel`` with n_jobs=1. See https://github.com/joblib/joblib/issues/483 for more details. Loïc Estève Fix race condition when a function cached with ``joblib.Memory.cache`` was used inside a ``joblib.Parallel``. See https://github.com/joblib/joblib/issues/490 for more details. Release 0.10.3 -------------- Loïc Estève Fix tests when multiprocessing is disabled via the JOBLIB_MULTIPROCESSING environment variable. harishmk Remove warnings in nested Parallel objects when the inner Parallel has n_jobs=1. See https://github.com/joblib/joblib/pull/406 for more details. Release 0.10.2 -------------- Loïc Estève FIX a bug in stack formatting when the error happens in a compiled extension. See https://github.com/joblib/joblib/pull/382 for more details. Vincent Latrouite FIX a bug in the constructor of BinaryZlibFile that would throw an exception when passing unicode filename (Python 2 only). See https://github.com/joblib/joblib/pull/384 for more details. Olivier Grisel Expose :class:`joblib.parallel.ParallelBackendBase` and :class:`joblib.parallel.AutoBatchingMixin` in the public API to make them officially re-usable by backend implementers. Release 0.10.0 -------------- Alexandre Abadie ENH: joblib.dump/load now accept file-like objects besides filenames. https://github.com/joblib/joblib/pull/351 for more details. Niels Zeilemaker and Olivier Grisel Refactored joblib.Parallel to enable the registration of custom computational backends. https://github.com/joblib/joblib/pull/306 Note the API to register custom backends is considered experimental and subject to change without deprecation. Alexandre Abadie Joblib pickle format change: joblib.dump always create a single pickle file and joblib.dump/joblib.save never do any memory copy when writing/reading pickle files. Reading pickle files generated with joblib versions prior to 0.10 will be supported for a limited amount of time, we advise to regenerate them from scratch when convenient. joblib.dump and joblib.load also support pickle files compressed using various strategies: zlib, gzip, bz2, lzma and xz. Note that lzma and xz are only available with python >= 3.3. https://github.com/joblib/joblib/pull/260 for more details. Antony Lee ENH: joblib.dump/load now accept pathlib.Path objects as filenames. https://github.com/joblib/joblib/pull/316 for more details. Olivier Grisel Workaround for "WindowsError: [Error 5] Access is denied" when trying to terminate a multiprocessing pool under Windows: https://github.com/joblib/joblib/issues/354 Release 0.9.4 ------------- Olivier Grisel FIX a race condition that could cause a joblib.Parallel to hang when collecting the result of a job that triggers an exception. https://github.com/joblib/joblib/pull/296 Olivier Grisel FIX a bug that caused joblib.Parallel to wrongly reuse previously memmapped arrays instead of creating new temporary files. https://github.com/joblib/joblib/pull/294 for more details. Loïc Estève FIX for raising non inheritable exceptions in a Parallel call. See https://github.com/joblib/joblib/issues/269 for more details. Alexandre Abadie FIX joblib.hash error with mixed types sets and dicts containing mixed types keys when using Python 3. see https://github.com/joblib/joblib/issues/254 Loïc Estève FIX joblib.dump/load for big numpy arrays with dtype=object. See https://github.com/joblib/joblib/issues/220 for more details. Loïc Estève FIX joblib.Parallel hanging when used with an exhausted iterator. See https://github.com/joblib/joblib/issues/292 for more details. Release 0.9.3 ------------- Olivier Grisel Revert back to the ``fork`` start method (instead of ``forkserver``) as the latter was found to cause crashes in interactive Python sessions. Release 0.9.2 ------------- Loïc Estève Joblib hashing now uses the default pickle protocol (2 for Python 2 and 3 for Python 3). This makes it very unlikely to get the same hash for a given object under Python 2 and Python 3. In particular, for Python 3 users, this means that the output of joblib.hash changes when switching from joblib 0.8.4 to 0.9.2 . We strive to ensure that the output of joblib.hash does not change needlessly in future versions of joblib but this is not officially guaranteed. Loïc Estève Joblib pickles generated with Python 2 can not be loaded with Python 3 and the same applies for joblib pickles generated with Python 3 and loaded with Python 2. During the beta period 0.9.0b2 to 0.9.0b4, we experimented with a joblib serialization that aimed to make pickles serialized with Python 3 loadable under Python 2. Unfortunately this serialization strategy proved to be too fragile as far as the long-term maintenance was concerned (For example see https://github.com/joblib/joblib/pull/243). That means that joblib pickles generated with joblib 0.9.0bN can not be loaded under joblib 0.9.2. Joblib beta testers, who are the only ones likely to be affected by this, are advised to delete their joblib cache when they upgrade from 0.9.0bN to 0.9.2. Arthur Mensch Fixed a bug with ``joblib.hash`` that used to return unstable values for strings and numpy.dtype instances depending on interning states. Olivier Grisel Make joblib use the 'forkserver' start method by default under Python 3.4+ to avoid causing crash with 3rd party libraries (such as Apple vecLib / Accelerate or the GCC OpenMP runtime) that use an internal thread pool that is not reinitialized when a ``fork`` system call happens. Olivier Grisel New context manager based API (``with`` block) to re-use the same pool of workers across consecutive parallel calls. Vlad Niculae and Olivier Grisel Automated batching of fast tasks into longer running jobs to hide multiprocessing dispatching overhead when possible. Olivier Grisel FIX make it possible to call ``joblib.load(filename, mmap_mode='r')`` on pickled objects that include a mix of arrays of both memory memmapable dtypes and object dtype. Release 0.8.4 ------------- 2014-11-20 Olivier Grisel OPTIM use the C-optimized pickler under Python 3 This makes it possible to efficiently process parallel jobs that deal with numerous Python objects such as large dictionaries. Release 0.8.3 ------------- 2014-08-19 Olivier Grisel FIX disable memmapping for object arrays 2014-08-07 Lars Buitinck MAINT NumPy 1.10-safe version comparisons 2014-07-11 Olivier Grisel FIX #146: Heisen test failure caused by thread-unsafe Python lists This fix uses a queue.Queue datastructure in the failing test. This datastructure is thread-safe thanks to an internal Lock. This Lock instance not picklable hence cause the picklability check of delayed to check fail. When using the threading backend, picklability is no longer required, hence this PRs give the user the ability to disable it on a case by case basis. Release 0.8.2 ------------- 2014-06-30 Olivier Grisel BUG: use mmap_mode='r' by default in Parallel and MemmappingPool The former default of mmap_mode='c' (copy-on-write) caused problematic use of the paging file under Windows. 2014-06-27 Olivier Grisel BUG: fix usage of the /dev/shm folder under Linux Release 0.8.1 ------------- 2014-05-29 Gael Varoquaux BUG: fix crash with high verbosity Release 0.8.0 ------------- 2014-05-14 Olivier Grisel Fix a bug in exception reporting under Python 3 2014-05-10 Olivier Grisel Fixed a potential segfault when passing non-contiguous memmap instances. 2014-04-22 Gael Varoquaux ENH: Make memory robust to modification of source files while the interpreter is running. Should lead to less spurious cache flushes and recomputations. 2014-02-24 Philippe Gervais New ``Memory.call_and_shelve`` API to handle memoized results by reference instead of by value. Release 0.8.0a3 --------------- 2014-01-10 Olivier Grisel & Gael Varoquaux FIX #105: Race condition in task iterable consumption when pre_dispatch != 'all' that could cause crash with error messages "Pools seems closed" and "ValueError: generator already executing". 2014-01-12 Olivier Grisel FIX #72: joblib cannot persist "output_dir" keyword argument. Release 0.8.0a2 --------------- 2013-12-23 Olivier Grisel ENH: set default value of Parallel's max_nbytes to 100MB Motivation: avoid introducing disk latency on medium sized parallel workload where memory usage is not an issue. FIX: properly handle the JOBLIB_MULTIPROCESSING env variable FIX: timeout test failures under windows Release 0.8.0a -------------- 2013-12-19 Olivier Grisel FIX: support the new Python 3.4 multiprocessing API 2013-12-05 Olivier Grisel ENH: make Memory respect mmap_mode at first call too ENH: add a threading based backend to Parallel This is low overhead alternative backend to the default multiprocessing backend that is suitable when calling compiled extensions that release the GIL. Author: Dan Stahlke Date: 2013-11-08 FIX: use safe_repr to print arg vals in trace This fixes a problem in which extremely long (and slow) stack traces would be produced when function parameters are large numpy arrays. 2013-09-10 Olivier Grisel ENH: limit memory copy with Parallel by leveraging numpy.memmap when possible Release 0.7.1 --------------- 2013-07-25 Gael Varoquaux MISC: capture meaningless argument (n_jobs=0) in Parallel 2013-07-09 Lars Buitinck ENH Handles tuples, sets and Python 3's dict_keys type the same as lists. in pre_dispatch 2013-05-23 Martin Luessi ENH: fix function caching for IPython Release 0.7.0 --------------- **This release drops support for Python 2.5 in favor of support for Python 3.0** 2013-02-13 Gael Varoquaux BUG: fix nasty hash collisions 2012-11-19 Gael Varoquaux ENH: Parallel: Turn of pre-dispatch for already expanded lists Gael Varoquaux 2012-11-19 ENH: detect recursive sub-process spawning, as when people do not protect the __main__ in scripts under Windows, and raise a useful error. Gael Varoquaux 2012-11-16 ENH: Full python 3 support Release 0.6.5 --------------- 2012-09-15 Yannick Schwartz BUG: make sure that sets and dictionaries give reproducible hashes 2012-07-18 Marek Rudnicki BUG: make sure that object-dtype numpy array hash correctly 2012-07-12 GaelVaroquaux BUG: Bad default n_jobs for Parallel Release 0.6.4 --------------- 2012-05-07 Vlad Niculae ENH: controlled randomness in tests and doctest fix 2012-02-21 GaelVaroquaux ENH: add verbosity in memory 2012-02-21 GaelVaroquaux BUG: non-reproducible hashing: order of kwargs The ordering of a dictionary is random. As a result the function hashing was not reproducible. Pretty hard to test Release 0.6.3 --------------- 2012-02-14 GaelVaroquaux BUG: fix joblib Memory pickling 2012-02-11 GaelVaroquaux BUG: fix hasher with Python 3 2012-02-09 GaelVaroquaux API: filter_args: `*args, **kwargs -> args, kwargs` Release 0.6.2 --------------- 2012-02-06 Gael Varoquaux BUG: make sure Memory pickles even if cachedir=None Release 0.6.1 --------------- Bugfix release because of a merge error in release 0.6.0 Release 0.6.0 --------------- **Beta 3** 2012-01-11 Gael Varoquaux BUG: ensure compatibility with old numpy DOC: update installation instructions BUG: file semantic to work under Windows 2012-01-10 Yaroslav Halchenko BUG: a fix toward 2.5 compatibility **Beta 2** 2012-01-07 Gael Varoquaux ENH: hash: bugware to be able to hash objects defined interactively in IPython 2012-01-07 Gael Varoquaux ENH: Parallel: warn and not fail for nested loops ENH: Parallel: n_jobs=-2 now uses all CPUs but one 2012-01-01 Juan Manuel Caicedo Carvajal and Gael Varoquaux ENH: add verbosity levels in Parallel Release 0.5.7 --------------- 2011-12-28 Gael varoquaux API: zipped -> compress 2011-12-26 Gael varoquaux ENH: Add a zipped option to Memory API: Memory no longer accepts save_npy 2011-12-22 Kenneth C. Arnold and Gael varoquaux BUG: fix numpy_pickle for array subclasses 2011-12-21 Gael varoquaux ENH: add zip-based pickling 2011-12-19 Fabian Pedregosa Py3k: compatibility fixes. This makes run fine the tests test_disk and test_parallel Release 0.5.6 --------------- 2011-12-11 Lars Buitinck ENH: Replace os.path.exists before makedirs with exception check New disk.mkdirp will fail with other errnos than EEXIST. 2011-12-10 Bala Subrahmanyam Varanasi MISC: pep8 compliant Release 0.5.5 --------------- 2011-19-10 Fabian Pedregosa ENH: Make joblib installable under Python 3.X Release 0.5.4 --------------- 2011-09-29 Jon Olav Vik BUG: Make mangling path to filename work on Windows 2011-09-25 Olivier Grisel FIX: doctest heisenfailure on execution time 2011-08-24 Ralf Gommers STY: PEP8 cleanup. Release 0.5.3 --------------- 2011-06-25 Gael varoquaux API: All the useful symbols in the __init__ Release 0.5.2 --------------- 2011-06-25 Gael varoquaux ENH: Add cpu_count 2011-06-06 Gael varoquaux ENH: Make sure memory hash in a reproducible way Release 0.5.1 --------------- 2011-04-12 Gael varoquaux TEST: Better testing of parallel and pre_dispatch Yaroslav Halchenko 2011-04-12 DOC: quick pass over docs -- trailing spaces/spelling Yaroslav Halchenko 2011-04-11 ENH: JOBLIB_MULTIPROCESSING env var to disable multiprocessing from the environment Alexandre Gramfort 2011-04-08 ENH : adding log message to know how long it takes to load from disk the cache Release 0.5.0 --------------- 2011-04-01 Gael varoquaux BUG: pickling MemoizeFunc does not store timestamp 2011-03-31 Nicolas Pinto TEST: expose hashing bug with cached method 2011-03-26...2011-03-27 Pietro Berkes BUG: fix error management in rm_subdirs BUG: fix for race condition during tests in mem.clear() Gael varoquaux 2011-03-22...2011-03-26 TEST: Improve test coverage and robustness Gael varoquaux 2011-03-19 BUG: hashing functions with only \*var \**kwargs Gael varoquaux 2011-02-01... 2011-03-22 BUG: Many fixes to capture interprocess race condition when mem.cache is used by several processes on the same cache. Fabian Pedregosa 2011-02-28 First work on Py3K compatibility Gael varoquaux 2011-02-27 ENH: pre_dispatch in parallel: lazy generation of jobs in parallel for to avoid drowning memory. GaelVaroquaux 2011-02-24 ENH: Add the option of overloading the arguments of the mother 'Memory' object in the cache method that is doing the decoration. Gael varoquaux 2010-11-21 ENH: Add a verbosity level for more verbosity Release 0.4.6 ---------------- Gael varoquaux 2010-11-15 ENH: Deal with interruption in parallel Gael varoquaux 2010-11-13 BUG: Exceptions raised by Parallel when n_job=1 are no longer captured. Gael varoquaux 2010-11-13 BUG: Capture wrong arguments properly (better error message) Release 0.4.5 ---------------- Pietro Berkes 2010-09-04 BUG: Fix Windows peculiarities with path separators and file names BUG: Fix more windows locking bugs Gael varoquaux 2010-09-03 ENH: Make sure that exceptions raised in Parallel also inherit from the original exception class ENH: Add a shadow set of exceptions Fabian Pedregosa 2010-09-01 ENH: Clean up the code for parallel. Thanks to Fabian Pedregosa for the patch. Release 0.4.4 ---------------- Gael varoquaux 2010-08-23 BUG: Fix Parallel on computers with only one CPU, for n_jobs=-1. Gael varoquaux 2010-08-02 BUG: Fix setup.py for extra setuptools args. Gael varoquaux 2010-07-29 MISC: Silence tests (and hopefully Yaroslav :P) Release 0.4.3 ---------------- Gael Varoquaux 2010-07-22 BUG: Fix hashing for function with a side effect modifying their input argument. Thanks to Pietro Berkes for reporting the bug and proving the patch. Release 0.4.2 ---------------- Gael Varoquaux 2010-07-16 BUG: Make sure that joblib still works with Python2.5. => release 0.4.2 Release 0.4.1 ---------------- joblib-1.3.2/LICENSE.txt000066400000000000000000000027671446465525000146300ustar00rootroot00000000000000BSD 3-Clause License Copyright (c) 2008-2021, The joblib developers. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. joblib-1.3.2/MANIFEST.in000066400000000000000000000002001446465525000145170ustar00rootroot00000000000000include *.txt *.py recursive-include joblib *.rst *.py graft doc graft doc/_static graft doc/_templates global-exclude *~ *.swp joblib-1.3.2/Makefile000066400000000000000000000005531446465525000144340ustar00rootroot00000000000000.PHONY: all test test-no-multiprocessing test-doc doc doc-clean all: test test: pytest joblib --timeout 30 -vl test-no-multiprocessing: export JOBLIB_MULTIPROCESSING=0 && pytest joblib test-doc: pytest $(shell find doc -name '*.rst' | sort) # generate html documentation using sphinx doc: make -C doc # clean documentation doc-clean: make -C doc clean joblib-1.3.2/README.rst000066400000000000000000000101621446465525000144600ustar00rootroot00000000000000|PyPi| |Azure| |ReadTheDocs| |Codecov| .. |PyPi| image:: https://badge.fury.io/py/joblib.svg :target: https://badge.fury.io/py/joblib :alt: Joblib version .. |Azure| image:: https://dev.azure.com/joblib/joblib/_apis/build/status/joblib.joblib?branchName=master :target: https://dev.azure.com/joblib/joblib/_build?definitionId=3&_a=summary&branchFilter=40 :alt: Azure CI status .. |ReadTheDocs| image:: https://readthedocs.org/projects/joblib/badge/?version=latest :target: https://joblib.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status .. |Codecov| image:: https://codecov.io/gh/joblib/joblib/branch/master/graph/badge.svg :target: https://codecov.io/gh/joblib/joblib :alt: Codecov coverage The homepage of joblib with user documentation is located on: https://joblib.readthedocs.io Getting the latest code ======================= To get the latest code using git, simply type:: git clone git://github.com/joblib/joblib.git If you don't have git installed, you can download a zip or tarball of the latest code: http://github.com/joblib/joblib/archives/master Installing ========== You can use `pip` to install joblib:: pip install joblib from any directory or:: python setup.py install from the source directory. Dependencies ============ - Joblib has no mandatory dependencies besides Python (supported versions are 3.7+). - Joblib has an optional dependency on Numpy (at least version 1.6.1) for array manipulation. - Joblib includes its own vendored copy of `loky `_ for process management. - Joblib can efficiently dump and load numpy arrays but does not require numpy to be installed. - Joblib has an optional dependency on `python-lz4 `_ as a faster alternative to zlib and gzip for compressed serialization. - Joblib has an optional dependency on psutil to mitigate memory leaks in parallel worker processes. - Some examples require external dependencies such as pandas. See the instructions in the `Building the docs`_ section for details. Workflow to contribute ====================== To contribute to joblib, first create an account on `github `_. Once this is done, fork the `joblib repository `_ to have your own repository, clone it using 'git clone' on the computers where you want to work. Make your changes in your clone, push them to your github account, test them on several computers, and when you are happy with them, send a pull request to the main repository. Running the test suite ====================== To run the test suite, you need the pytest (version >= 3) and coverage modules. Run the test suite using:: pytest joblib from the root of the project. Building the docs ================= To build the docs you need to have sphinx (>=1.4) and some dependencies installed:: pip install -U -r .readthedocs-requirements.txt The docs can then be built with the following command:: make doc The html docs are located in the ``doc/_build/html`` directory. Making a source tarball ======================= To create a source tarball, eg for packaging or distributing, run the following command:: python setup.py sdist The tarball will be created in the `dist` directory. This command will compile the docs, and the resulting tarball can be installed with no extra dependencies than the Python standard library. You will need setuptool and sphinx. Making a release and uploading it to PyPI ========================================= This command is only run by project manager, to make a release, and upload in to PyPI:: python setup.py sdist bdist_wheel twine upload dist/* Note that the documentation should automatically get updated at each git push. If that is not the case, try building th doc locally and resolve any doc build error (in particular when running the examples). Updating the changelog ====================== Changes are listed in the CHANGES.rst file. They must be manually updated but, the following git command may be used to generate the lines:: git log --abbrev-commit --date=short --no-merges --sparse joblib-1.3.2/TODO.rst000066400000000000000000000032571446465525000142770ustar00rootroot00000000000000Tasks at hand on joblib, in increasing order of difficulty. * Add a changelog! * In parallel: need to deal with return arguments that don't pickle. * Improve test coverage and documentation * Store a repr of the arguments for each call in the corresponding cachedir * Try to use Mike McKerns's Dill pickling module in Parallel: Implementation idea: * Create a new function that is wrapped and takes Dillo pickles as inputs as output, feed this one to multiprocessing * pickle everything using Dill in the Parallel object. http://dev.danse.us/trac/pathos/browser/dill * Make a sensible error message when wrong keyword arguments are given, currently we have:: from joblib import Memory mem = Memory(cachedir='cache') def f(a=0, b=2): return a, b g = mem.cache(f) g(c=2) /home/varoquau/dev/joblib/joblib/func_inspect.pyc in filter_args(func, ignore_lst, *args, **kwargs), line 168 TypeError: Ignore list for diffusion_reorder() contains and unexpected keyword argument 'cachedir' * add a 'depends' keyword argument to memory.cache, to be able to specify that a function depends on other functions, and thus that the cache should be cleared. * add a 'argument_hash' keyword argument to Memory.cache, to be able to replace the hashing logic of memory for the input arguments. It should accept as an input the dictionary of arguments, as returned in func_inspect, and return a string. * add a sqlite db for provenance tracking. Store computation time and usage timestamps, to be able to do 'garbage-collection-like' cleaning of unused results, based on a cost function balancing computation cost and frequency of use. joblib-1.3.2/azure-pipelines.yml000066400000000000000000000075631446465525000166430ustar00rootroot00000000000000# Python package # Create and test a Python package on multiple Python versions. # Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: # https://docs.microsoft.com/azure/devops/pipelines/languages/python schedules: - cron: "0 9 * * *" displayName: Daily build branches: include: - master trigger: - master jobs: - job: linting displayName: Linting pool: vmImage: ubuntu-latest steps: - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH - bash: sudo chown -R $USER $CONDA displayName: Take ownership of conda installation - bash: conda create --name flake8_env --yes flake8 displayName: Install flake8 - bash: | if [[ $BUILD_SOURCEVERSIONMESSAGE =~ \[lint\ skip\] ]]; then # skip linting echo "Skipping linting" exit 0 else source activate flake8_env flake8 fi displayName: Run linting - job: testing displayName: Testing strategy: matrix: linux_pypy3: imageName: 'ubuntu-latest' PYTHON_VERSION: "pypy3" LOKY_MAX_CPU_COUNT: "2" linux_py39_sklearn_tests: imageName: 'ubuntu-latest' PYTHON_VERSION: "3.9" # SKIP_TESTS: "true" SKLEARN_TESTS: "true" linux_py310_distributed: # To be updated regularly to use the most recent versions of the # dependencies. imageName: 'ubuntu-latest' PYTHON_VERSION: "3.10" EXTRA_CONDA_PACKAGES: "numpy=1.23 distributed=2022.2.0" linux_py37_distributed: imageName: 'ubuntu-latest' PYTHON_VERSION: "3.7" EXTRA_CONDA_PACKAGES: "numpy=1.15 distributed=2.13" linux_py311_cython: imageName: 'ubuntu-latest' PYTHON_VERSION: "3.11" EXTRA_CONDA_PACKAGES: "numpy=1.24.1" CYTHON: "true" linux_py37_no_multiprocessing_no_lzma: imageName: 'ubuntu-latest' PYTHON_VERSION: "3.7" EXTRA_CONDA_PACKAGES: "numpy=1.15" JOBLIB_MULTIPROCESSING: "0" NO_LZMA: "1" linux_py37_no_numpy: imageName: 'ubuntu-latest' PYTHON_VERSION: "3.7" linux_py37_default_backend_threading: imageName: 'ubuntu-latest' PYTHON_VERSION: "3.7" JOBLIB_TESTS_DEFAULT_PARALLEL_BACKEND: "threading" windows_py310: imageName: "windows-latest" PYTHON_VERSION: "3.10" EXTRA_CONDA_PACKAGES: "numpy=1.23" macos_py310: imageName: "macos-latest" PYTHON_VERSION: "3.10" EXTRA_CONDA_PACKAGES: "numpy=1.23" macos_py37_no_numpy: imageName: "macos-latest" PYTHON_VERSION: "3.7" variables: JUNITXML: 'test-data.xml' COVERAGE: "true" pool: vmImage: $(imageName) steps: - bash: echo "##vso[task.prependpath]C:/Program Files/Git/bin" displayName: 'Override Git bash shell for Windows' condition: eq(variables['Agent.OS'], 'Windows_NT') - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" displayName: Add conda to PATH condition: eq(variables['Agent.OS'], 'Windows_NT') - bash: | echo "##vso[task.prependpath]$CONDA/bin" sudo chown -R $USER $CONDA displayName: Add conda to PATH condition: ne(variables['Agent.OS'], 'Windows_NT') - bash: bash continuous_integration/install.sh displayName: 'Install joblib and its dependencies' - bash: bash continuous_integration/run_tests.sh displayName: 'Run the tests' - task: PublishTestResults@2 inputs: testResultsFiles: '$(JUNITXML)' displayName: 'Publish Test Results' condition: and(succeededOrFailed(), ne(variables['SKIP_TESTS'], 'true')) - bash: curl -s https://codecov.io/bash | bash displayName: 'Upload to codecov' condition: and(succeeded(), ne(variables['SKIP_TESTS'], 'true'), eq(variables['COVERAGE'], 'true')) joblib-1.3.2/benchmarks/000077500000000000000000000000001446465525000151065ustar00rootroot00000000000000joblib-1.3.2/benchmarks/bench_auto_batching.py000066400000000000000000000103531446465525000214300ustar00rootroot00000000000000"""Benchmark batching="auto" on high number of fast tasks The goal of this script is to study the behavior of the batch_size='auto' and in particular the impact of the default value of the joblib.parallel.MIN_IDEAL_BATCH_DURATION constant. """ # Author: Olivier Grisel # License: BSD 3 clause import numpy as np import time import tempfile from pprint import pprint from joblib import Parallel, delayed from joblib._parallel_backends import AutoBatchingMixin def sleep_noop(duration, input_data, output_data_size): """Noop function to emulate real computation. Simulate CPU time with by sleeping duration. Induce overhead by accepting (and ignoring) any amount of data as input and allocating a requested amount of data. """ time.sleep(duration) if output_data_size: return np.ones(output_data_size, dtype=np.byte) def bench_short_tasks(task_times, n_jobs=2, batch_size="auto", pre_dispatch="2*n_jobs", verbose=True, input_data_size=0, output_data_size=0, backend=None, memmap_input=False): with tempfile.NamedTemporaryFile() as temp_file: if input_data_size: # Generate some input data with the required size if memmap_input: temp_file.close() input_data = np.memmap(temp_file.name, shape=input_data_size, dtype=np.byte, mode='w+') input_data[:] = 1 else: input_data = np.ones(input_data_size, dtype=np.byte) else: input_data = None t0 = time.time() p = Parallel(n_jobs=n_jobs, verbose=verbose, pre_dispatch=pre_dispatch, batch_size=batch_size, backend=backend) p(delayed(sleep_noop)(max(t, 0), input_data, output_data_size) for t in task_times) duration = time.time() - t0 effective_batch_size = getattr(p._backend, '_effective_batch_size', p.batch_size) print('Completed {} tasks in {:3f}s, final batch_size={}\n'.format( len(task_times), duration, effective_batch_size)) return duration, effective_batch_size if __name__ == "__main__": bench_parameters = dict( # batch_size=200, # batch_size='auto' by default # memmap_input=True, # if True manually memmap input out of timing # backend='threading', # backend='multiprocessing' by default # pre_dispatch='n_jobs', # pre_dispatch="2*n_jobs" by default input_data_size=int(2e7), # input data size in bytes output_data_size=int(1e5), # output data size in bytes n_jobs=2, verbose=10, ) print("Common benchmark parameters:") pprint(bench_parameters) AutoBatchingMixin.MIN_IDEAL_BATCH_DURATION = 0.2 AutoBatchingMixin.MAX_IDEAL_BATCH_DURATION = 2 # First pair of benchmarks to check that the auto-batching strategy is # stable (do not change the batch size too often) in the presence of large # variance while still be comparable to the equivalent load without # variance print('# high variance, no trend') # censored gaussian distribution high_variance = np.random.normal(loc=0.000001, scale=0.001, size=5000) high_variance[high_variance < 0] = 0 bench_short_tasks(high_variance, **bench_parameters) print('# low variance, no trend') low_variance = np.empty_like(high_variance) low_variance[:] = np.mean(high_variance) bench_short_tasks(low_variance, **bench_parameters) # Second pair of benchmarks: one has a cycling task duration pattern that # the auto batching feature should be able to roughly track. We use an even # power of cos to get only positive task durations with a majority close to # zero (only data transfer overhead). The shuffle variant should not # oscillate too much and still approximately have the same total run time. print('# cyclic trend') slow_time = 0.1 positive_wave = np.cos(np.linspace(1, 4 * np.pi, 300)) ** 8 cyclic = positive_wave * slow_time bench_short_tasks(cyclic, **bench_parameters) print("shuffling of the previous benchmark: same mean and variance") np.random.shuffle(cyclic) bench_short_tasks(cyclic, **bench_parameters) joblib-1.3.2/benchmarks/bench_compression.py000066400000000000000000000205601446465525000211630ustar00rootroot00000000000000"""Script comparing different pickling strategies.""" from joblib.numpy_pickle import NumpyPickler, NumpyUnpickler from joblib.numpy_pickle_utils import BinaryZlibFile, BinaryGzipFile from pickle import _Pickler, _Unpickler, Pickler, Unpickler import numpy as np import bz2 import lzma import time import io import sys import os from collections import OrderedDict def fileobj(obj, fname, mode, kwargs): """Create a file object.""" return obj(fname, mode, **kwargs) def bufferize(f, buf): """Bufferize a fileobject using buf.""" if buf is None: return f else: if (buf.__name__ == io.BufferedWriter.__name__ or buf.__name__ == io.BufferedReader.__name__): return buf(f, buffer_size=10 * 1024 ** 2) return buf(f) def _load(unpickler, fname, f): if unpickler.__name__ == NumpyUnpickler.__name__: p = unpickler(fname, f) else: p = unpickler(f) return p.load() def print_line(obj, strategy, buffer, pickler, dump, load, disk_used): """Nice printing function.""" print('% 20s | %6s | % 14s | % 7s | % 5.1f | % 5.1f | % 5s' % ( obj, strategy, buffer, pickler, dump, load, disk_used)) class PickleBufferedWriter(): """Protect the underlying fileobj against numerous calls to write This is achieved by internally keeping a list of small chunks and only flushing to the backing fileobj if passed a large chunk or after a threshold on the number of small chunks. """ def __init__(self, fileobj, max_buffer_size=10 * 1024 ** 2): self._fileobj = fileobj self._chunks = chunks = [] # As the `write` method is called many times by the pickler, # attribute look ups on the self's __dict__ are too expensive # hence we define a closure here with all the regularly # accessed parameters def _write(data): chunks.append(data) if len(chunks) > max_buffer_size: self.flush() self.write = _write def flush(self): self._fileobj.write(b''.join(self._chunks[:])) del self._chunks[:] def close(self): self.flush() self._fileobj.close() def __enter__(self): return self def __exit__(self, *exc): self.close() return False class PickleBufferedReader(): """Protect the underlying fileobj against numerous calls to write This is achieved by internally keeping a list of small chunks and only flushing to the backing fileobj if passed a large chunk or after a threshold on the number of small chunks. """ def __init__(self, fileobj, max_buffer_size=10 * 1024 ** 2): self._fileobj = fileobj self._buffer = bytearray(max_buffer_size) self.max_buffer_size = max_buffer_size self._position = 0 def read(self, n=None): data = b'' if n is None: data = self._fileobj.read() else: while len(data) < n: if self._position == 0: self._buffer = self._fileobj.read(self.max_buffer_size) elif self._position == self.max_buffer_size: self._position = 0 continue next_position = min(self.max_buffer_size, self._position + n - len(data)) data += self._buffer[self._position:next_position] self._position = next_position return data def readline(self): line = [] while True: c = self.read(1) line.append(c) if c == b'\n': break return b''.join(line) def close(self): self._fileobj.close() def __enter__(self): return self def __exit__(self, *exc): self.close() return False def run_bench(): print('% 20s | %10s | % 12s | % 8s | % 9s | % 9s | % 5s' % ( 'Object', 'Compression', 'Buffer', 'Pickler/Unpickler', 'dump time (s)', 'load time (s)', 'Disk used (MB)')) print("--- | --- | --- | --- | --- | --- | ---") for oname, obj in objects.items(): # Looping over the objects (array, dict, etc) if isinstance(obj, np.ndarray): osize = obj.nbytes / 1e6 else: osize = sys.getsizeof(obj) / 1e6 for cname, f in compressors.items(): fobj = f[0] fname = f[1] fmode = f[2] fopts = f[3] # Looping other defined compressors for bname, buf in bufs.items(): writebuf = buf[0] readbuf = buf[1] # Looping other picklers for pname, p in picklers.items(): pickler = p[0] unpickler = p[1] t0 = time.time() # Now pickling the object in the file if (writebuf is not None and writebuf.__name__ == io.BytesIO.__name__): b = writebuf() p = pickler(b) p.dump(obj) with fileobj(fobj, fname, fmode, fopts) as f: f.write(b.getvalue()) else: with bufferize(fileobj(fobj, fname, fmode, fopts), writebuf) as f: p = pickler(f) p.dump(obj) dtime = time.time() - t0 t0 = time.time() # Now loading the object from the file obj_r = None if (readbuf is not None and readbuf.__name__ == io.BytesIO.__name__): b = readbuf() with fileobj(fobj, fname, 'rb', {}) as f: b.write(f.read()) b.seek(0) obj_r = _load(unpickler, fname, b) else: with bufferize(fileobj(fobj, fname, 'rb', {}), readbuf) as f: obj_r = _load(unpickler, fname, f) ltime = time.time() - t0 if isinstance(obj, np.ndarray): assert (obj == obj_r).all() else: assert obj == obj_r print_line("{} ({:.1f}MB)".format(oname, osize), cname, bname, pname, dtime, ltime, "{:.2f}".format(os.path.getsize(fname) / 1e6)) # Defining objects used in this bench DICT_SIZE = int(1e6) ARRAY_SIZE = int(1e7) arr = np.random.normal(size=(ARRAY_SIZE)) arr[::2] = 1 # Objects used for testing objects = OrderedDict([ ("dict", dict((i, str(i)) for i in range(DICT_SIZE))), ("list", [i for i in range(DICT_SIZE)]), ("array semi-random", arr), ("array random", np.random.normal(size=(ARRAY_SIZE))), ("array ones", np.ones((ARRAY_SIZE))), ]) #  We test 3 different picklers picklers = OrderedDict([ # Python implementation of Pickler/Unpickler ("Pickle", (_Pickler, _Unpickler)), # C implementation of Pickler/Unpickler ("cPickle", (Pickler, Unpickler)), # Joblib Pickler/Unpickler designed for numpy arrays. ("Joblib", (NumpyPickler, NumpyUnpickler)), ]) # The list of supported compressors used for testing compressors = OrderedDict([ ("No", (open, '/tmp/test_raw', 'wb', {})), ("Zlib", (BinaryZlibFile, '/tmp/test_zlib', 'wb', {'compresslevel': 3})), ("Gzip", (BinaryGzipFile, '/tmp/test_gzip', 'wb', {'compresslevel': 3})), ("Bz2", (bz2.BZ2File, '/tmp/test_bz2', 'wb', {'compresslevel': 3})), ("Xz", (lzma.LZMAFile, '/tmp/test_xz', 'wb', {'preset': 3, 'check': lzma.CHECK_NONE})), ("Lzma", (lzma.LZMAFile, '/tmp/test_lzma', 'wb', {'preset': 3, 'format': lzma.FORMAT_ALONE})), ]) # Test 3 buffering strategies bufs = OrderedDict([ ("None", (None, None)), ("io.BytesIO", (io.BytesIO, io.BytesIO)), ("io.Buffered", (io.BufferedWriter, io.BufferedReader)), ("PickleBuffered", (PickleBufferedWriter, PickleBufferedReader)), ]) if __name__ == "__main__": run_bench() joblib-1.3.2/benchmarks/bench_grid_search_scaling.py000066400000000000000000000063141446465525000225750ustar00rootroot00000000000000"""Benchmark a small scale scikit-learn GridSearch and the scaling with n_jobs. This benchmark requires ``scikit-learn`` to be installed. The goal of this script is to make sure the scaling does not worsen with time. In particular, it can be used to compare 2 joblib versions by first running the benchmark with the option `-n name1`, then changing the joblib version and running the script with option `-c name1`. This option can be used multiple times to build a comparison with more than 2 version. """ from time import time import matplotlib.pyplot as plt import joblib import numpy as np from sklearn.svm import SVC from sklearn import datasets from sklearn.model_selection import GridSearchCV def get_file_name(name): return f"bench_gs_scaling_{name}.npy" if __name__ == "__main__": import argparse parser = argparse.ArgumentParser( description="") parser.add_argument( "--n-rep", "-r", type=int, default=5, help="Number of repetition to average on." ) parser.add_argument( "--name", "-n", type=str, default="", help="Name to save the results with. This can be used to compare " "different branches with '-c'." ) parser.add_argument( "--compare", "-c", action="append", help="Loads the results from a benchmark saved previously with a name " "given as the present argument value. This allows comparing the " "results across different versions of joblib." ) args = parser.parse_args() # Generate a synthetic dataset for classification. rng = np.random.RandomState(0) X, y = datasets.make_classification(n_samples=1000, random_state=rng) # gammas = [1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7] Cs = [1, 10, 100, 1e3, 1e4, 1e5] param_grid = {"gamma": gammas, "C": Cs} clf = SVC(random_state=rng) # Warm up run to avoid the first run overhead of starting the executor. GridSearchCV(estimator=clf, param_grid=param_grid, n_jobs=-1).fit(X, y) # We run the n_jobs in decreasing order to avoid the issue joblib/loky#396 # that make the queue size too small when increasing an executor size. res = [] for n_jobs in range(joblib.cpu_count(), 0, -2): T = [] for _ in range(args.n_rep): tic = time() gs = GridSearchCV( estimator=clf, param_grid=param_grid, n_jobs=n_jobs ) gs.fit(X, y) T += [time() - tic] res += [(n_jobs, *np.quantile(T, [0.5, 0.2, 0.8]))] res = np.array(res).T if args.name: fname = get_file_name(args.name) np.save(fname, res) label = args.name or "current" plt.fill_between(res[0], res[2], res[3], alpha=0.3, color="C0") plt.plot(res[0], res[1], c="C0", lw=2, label=label) if args.compare: for i, name_c in enumerate(args.compare): fname_compare = get_file_name(name_c) res_c = np.load(fname_compare) plt.fill_between( res_c[0], res_c[2], res_c[3], alpha=0.3, color=f"C{i+1}" ) plt.plot(res_c[0], res_c[1], c=f"C{i+1}", lw=2, label=name_c) plt.xlabel("n_jobs") plt.ylabel("Time [s]") plt.ylim(0, None) plt.legend() plt.show() joblib-1.3.2/benchmarks/bench_pickle.py000077500000000000000000000415341446465525000201000ustar00rootroot00000000000000""" Benching joblib pickle I/O. Warning: this is slow, and the benches are easily offset by other disk activity. """ import os import time import shutil import numpy as np import joblib import gc from joblib.disk import disk_used try: from memory_profiler import memory_usage except ImportError: memory_usage = None def clear_out(): """Clear output directory.""" if os.path.exists('out'): shutil.rmtree('out') os.mkdir('out') def kill_disk_cache(): """Clear disk cache to avoid side effects.""" if os.name == 'posix' and os.uname()[0] == 'Linux': try: os.system('sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches"') except IOError as e: if e.errno == 13: print('Please run me as root') else: raise else: # Write ~100M to the disk open('tmp', 'wb').write(np.random.random(2e7)) def delete_obj(obj): """Force destruction of an object.""" if obj is not None: del obj gc.collect() def memory_used(func, *args, **kwargs): """Compute memory usage of func.""" if memory_usage is None: return np.NaN gc.collect() mem_use = memory_usage((func, args, kwargs), interval=.001) return max(mem_use) - min(mem_use) def timeit(func, *args, **kwargs): """Compute the mean execution time of func based on 7 measures.""" times = [] tries = kwargs['tries'] kwargs.pop('tries') if tries > 1: tries += 2 for _ in range(tries): kill_disk_cache() t0 = time.time() out = func(*args, **kwargs) if 1: # Just time the function t1 = time.time() times.append(t1 - t0) else: # Compute a hash of the output, to estimate the time # necessary to access the elements: this is a better # estimate of the time to load with me mmapping. joblib.hash(out) t1 = time.time() joblib.hash(out) t2 = time.time() times.append(t2 - t0 - 2 * (t2 - t1)) times.sort() return np.mean(times[1:-1]) if tries > 1 else t1 - t0, out def generate_rand_dict(size, with_arrays=False, with_string=False, array_shape=(10, 10)): """Generate dictionary with random values from list of keys.""" ret = {} rnd = np.random.RandomState(0) randoms = rnd.random_sample((size)) for key, random in zip(range(size), randoms): if with_arrays: ret[str(key)] = rnd.random_sample(array_shape) elif with_string: ret[str(key)] = str(random) else: ret[str(key)] = int(random) return ret def generate_rand_list(size, with_arrays=False, with_string=False, array_shape=(10, 10)): """Generate list with random values from list of keys.""" ret = [] rnd = np.random.RandomState(0) for random in rnd.random_sample((size)): if with_arrays: ret.append(rnd.random_sample(array_shape)) elif with_string: ret.append(str(random)) else: ret.append(int(random)) return ret def print_line(dataset, strategy, write_time, read_time, mem_write, mem_read, disk_used): """Nice printing function.""" print('% 15s, %12s, % 6.3f, % 7.4f, % 9.1f, % 9.1f, % 5.1f' % ( dataset, strategy, write_time, read_time, mem_write, mem_read, disk_used)) def print_bench_summary(args): """Nice bench summary function.""" summary = """Benchmark summary: - Global values: . Joblib version: {} . Number of tries to compute mean execution time: {} . Compression levels : {} . Compression algorithm: {} . Memory map mode : {} . Bench nifti data : {} . Bench big array : {} . Bench 2 big arrays : {} . Bench big dictionary: {} . Bench array+dict : {} """.format(joblib.__version__, args.tries, ", ".join(map(str, args.compress)), "None" if not args.compress else args.compressor, args.mmap, args.nifti, args.array, args.arrays, args.dict, args.combo) if args.array: shape = tuple(args.shape) size = round(np.multiply.reduce(shape) * 8 / 1024 ** 2, 1) summary += """ - Big array: . shape: {} . size in memory: {} MB """.format(str(shape), size) if args.dict: summary += """ - Big dictionary: . number of keys: {} . value type: {} """.format(args.size, 'np.ndarray' if args.valuearray else 'str' if args.valuestring else 'int') if args.valuearray: summary += """ . arrays shape: {} """.format(str(tuple(args.valuearrayshape))) if args.list: summary += """ - Big list: . number of elements: {} . value type: {} """.format(args.size, 'np.ndarray' if args.valuearray else 'str' if args.valuestring else 'int') if args.valuearray: summary += """ . arrays shape: {} """.format(str(tuple(args.valuearrayshape))) print(summary) def bench_compress(dataset, name='', compress=('zlib', 0), cache_size=0, tries=5): """Bench joblib dump and load functions, compress modes.""" # generate output compression strategy string before joblib compatibility # check as it may override the compress variable with a non tuple type. compress_str = "Raw" if compress[1] == 0 else "{} {}".format(*compress) # joblib versions prior to 0.10 doesn't support tuple in compress argument # so only the second element of the tuple is used for those versions # and the compression strategy is ignored. if (isinstance(compress, tuple) and tuple(map(int, joblib.__version__.split('.')[:2])) < (0, 10)): compress = compress[1] time_write = time_read = du = mem_read = mem_write = [] clear_out() time_write, obj = timeit(joblib.dump, dataset, 'out/test.pkl', tries=tries, compress=compress, cache_size=cache_size) del obj gc.collect() mem_write = memory_used(joblib.dump, dataset, 'out/test.pkl', compress=compress, cache_size=cache_size) delete_obj(dataset) du = disk_used('out') / 1024. time_read, obj = timeit(joblib.load, 'out/test.pkl', tries=tries) delete_obj(obj) mem_read = memory_used(joblib.load, 'out/test.pkl') print_line(name, compress_str, time_write, time_read, mem_write, mem_read, du) def bench_mmap(dataset, name='', cache_size=0, mmap_mode='r', tries=5): """Bench joblib dump and load functions, memmap modes.""" time_write = time_read = du = [] clear_out() time_write, _ = timeit(joblib.dump, dataset, 'out/test.pkl', tries=tries, cache_size=cache_size) mem_write = memory_used(joblib.dump, dataset, 'out/test.pkl', cache_size=cache_size) delete_obj(dataset) time_read, obj = timeit(joblib.load, 'out/test.pkl', tries=tries, mmap_mode=mmap_mode) delete_obj(obj) mem_read = memory_used(joblib.load, 'out/test.pkl', mmap_mode=mmap_mode) du = disk_used('out') / 1024. print_line(name, 'mmap %s' % mmap_mode, time_write, time_read, mem_write, mem_read, du) def run_bench(func, obj, name, **kwargs): """Run the benchmark function.""" func(obj, name, **kwargs) def run(args): """Run the full bench suite.""" if args.summary: print_bench_summary(args) if (not args.nifti and not args.array and not args.arrays and not args.dict and not args.list and not args.combo): print("Nothing to bench. Exiting") return compress_levels = args.compress compress_method = args.compressor mmap_mode = args.mmap container_size = args.size a1_shape = tuple(args.shape) a2_shape = (10000000, ) print('% 15s, %12s, % 6s, % 7s, % 9s, % 9s, % 5s' % ( 'Dataset', 'strategy', 'write', 'read', 'mem_write', 'mem_read', 'disk')) if args.nifti: # Nifti images try: import nibabel except ImportError: print("nibabel is not installed skipping nifti file benchmark.") else: def load_nii(filename): img = nibabel.load(filename) return img.get_data(), img.get_affine() for name, nifti_file in ( ('MNI', '/usr/share/fsl/data/atlases' '/MNI/MNI-prob-1mm.nii.gz'), ('Juelich', '/usr/share/fsl/data/atlases' '/Juelich/Juelich-prob-2mm.nii.gz'), ): for c_order in (True, False): name_d = '% 5s(%s)' % (name, 'C' if c_order else 'F') for compress_level in compress_levels: d = load_nii(nifti_file) if c_order: d = (np.ascontiguousarray(d[0]), d[1]) run_bench(bench_compress, d, name_d, compress=(compress_method, compress_level), tries=args.tries) del d if not args.nommap: d = load_nii(nifti_file) if c_order: d = (np.ascontiguousarray(d[0]), d[1]) run_bench(bench_mmap, d, name_d, mmap_mode=mmap_mode, tries=args.tries) del d # Generate random seed rnd = np.random.RandomState(0) if args.array: # numpy array name = '% 5s' % 'Big array' for compress_level in compress_levels: a1 = rnd.random_sample(a1_shape) run_bench(bench_compress, a1, name, compress=(compress_method, compress_level), tries=args.tries) del a1 if not args.nommap: a1 = rnd.random_sample(a1_shape) run_bench(bench_mmap, a1, name, mmap_mode=mmap_mode, tries=args.tries) del a1 if args.arrays: # Complex object with 2 big arrays name = '% 5s' % '2 big arrays' for compress_level in compress_levels: obj = [rnd.random_sample(a1_shape), rnd.random_sample(a2_shape)] run_bench(bench_compress, obj, name, compress=(compress_method, compress_level), tries=args.tries) del obj if not args.nommap: obj = [rnd.random_sample(a1_shape), rnd.random_sample(a2_shape)] run_bench(bench_mmap, obj, name, mmap_mode=mmap_mode, tries=args.tries) del obj if args.dict: # Big dictionary name = '% 5s' % 'Big dict' array_shape = tuple(args.valuearrayshape) for compress_level in compress_levels: big_dict = generate_rand_dict(container_size, with_arrays=args.valuearray, with_string=args.valuestring, array_shape=array_shape) run_bench(bench_compress, big_dict, name, compress=(compress_method, compress_level), tries=args.tries) del big_dict if not args.nommap: big_dict = generate_rand_dict(container_size, with_arrays=args.valuearray, with_string=args.valuestring, array_shape=array_shape) run_bench(bench_mmap, big_dict, name, mmap_mode=mmap_mode, tries=args.tries) del big_dict if args.list: # Big dictionary name = '% 5s' % 'Big list' array_shape = tuple(args.valuearrayshape) for compress_level in compress_levels: big_list = generate_rand_list(container_size, with_arrays=args.valuearray, with_string=args.valuestring, array_shape=array_shape) run_bench(bench_compress, big_list, name, compress=(compress_method, compress_level), tries=args.tries) del big_list if not args.nommap: big_list = generate_rand_list(container_size, with_arrays=args.valuearray, with_string=args.valuestring, array_shape=array_shape) run_bench(bench_mmap, big_list, name, mmap_mode=mmap_mode, tries=args.tries) del big_list if args.combo: # 2 big arrays with one big dict name = '% 5s' % 'Dict/arrays' array_shape = tuple(args.valuearrayshape) for compress_level in compress_levels: obj = [rnd.random_sample(a1_shape), generate_rand_dict(container_size, with_arrays=args.valuearray, with_string=args.valuestring, array_shape=array_shape), rnd.random_sample(a2_shape)] run_bench(bench_compress, obj, name, compress=(compress_method, compress_level), tries=args.tries) del obj if not args.nommap: obj = [rnd.random_sample(a1_shape), generate_rand_dict(container_size, with_arrays=args.valuearray, with_string=args.valuestring, array_shape=array_shape), rnd.random_sample(a2_shape)] run_bench(bench_mmap, obj, name, mmap_mode=mmap_mode, tries=args.tries) del obj if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description="Joblib benchmark script") parser.add_argument('--compress', nargs='+', type=int, default=(0, 3), help="List of compress levels.") parser.add_argument('--compressor', type=str, default='zlib', choices=['zlib', 'gzip', 'bz2', 'xz', 'lzma'], help="Compression algorithm.") parser.add_argument('--mmap', type=str, default='r', choices=['r', 'r+', 'w+'], help="Memory map mode.") parser.add_argument('--tries', type=int, default=5, help="Number of tries to compute execution time" "mean on.") parser.add_argument('--shape', nargs='+', type=int, default=(10000, 10000), help="Big array shape.") parser.add_argument("-m", "--nommap", action="store_true", help="Don't bench memmap") parser.add_argument('--size', type=int, default=10000, help="Big dictionary size.") parser.add_argument('--valuearray', action="store_true", help="Use numpy arrays type in containers " "(list, dict)") parser.add_argument('--valuearrayshape', nargs='+', type=int, default=(10, 10), help="Shape of arrays in big containers.") parser.add_argument('--valuestring', action="store_true", help="Use string type in containers (list, dict).") parser.add_argument("-n", "--nifti", action="store_true", help="Benchmark Nifti data") parser.add_argument("-a", "--array", action="store_true", help="Benchmark single big numpy array") parser.add_argument("-A", "--arrays", action="store_true", help="Benchmark list of big numpy arrays") parser.add_argument("-d", "--dict", action="store_true", help="Benchmark big dictionary.") parser.add_argument("-l", "--list", action="store_true", help="Benchmark big list.") parser.add_argument("-c", "--combo", action="store_true", help="Benchmark big dictionary + list of " "big numpy arrays.") parser.add_argument("-s", "--summary", action="store_true", help="Show bench summary.") run(parser.parse_args()) joblib-1.3.2/benchmarks/bench_sequential_fast_tasks.py000066400000000000000000000062341446465525000232200ustar00rootroot00000000000000"""Benchmark n_jobs=1 on high number of fast tasks The goal of this script is to study the overhead incurred when calling small tasks with `n_jobs=1` compared to just running a simple list comprehension. """ # Author: Thomas Moreau # License: BSD 3 clause import time import numpy as np import pandas as pd import matplotlib.pyplot as plt from matplotlib.colors import LogNorm from matplotlib.cm import ScalarMappable from joblib import Parallel, delayed # Style for plots LINE_STYLES = {'iter': '--', 'parallel': '-', 'loop': ':'} COLORS = {'none': 'indianred'} CMAP = plt.colormaps['viridis'] # Generate functions that are more and more complex, to see # the relative impact depending on the task complexity funcs = [("none", lambda x: None, None)] n_size = 3 for i, n in enumerate(np.logspace(0, 2, n_size, dtype=int)): n = max(1, n) label = f'mat({n:3d}, {n:3d})' A = np.random.randn(n, n) funcs.append((label, lambda A: A @ A, A)) COLORS[label] = CMAP(i / (n_size - 1)) # For each function and for different number of repetition, # time the Parallel call. results = [] for f_name, func, arg in funcs: print('Benchmarking:', f_name) f_delayed = delayed(func) for N in np.logspace(1, 4, 4, dtype=int): print('# tasks:', N) for _ in range(10): t_start = time.perf_counter() list(func(arg) for _ in range(N)) runtime = time.perf_counter() - t_start results.append(dict( method="iter", N=N, func=f_name, runtime=runtime / N )) t_start = time.perf_counter() Parallel(n_jobs=1)(f_delayed(arg) for _ in range(N)) runtime = time.perf_counter() - t_start results.append(dict( method="parallel", N=N, func=f_name, runtime=runtime / N )) # Use a DataFrame to manipulate the results. df = pd.DataFrame(results) # Compute median runtime for each set of parameters curve = df.groupby(["method", "N", "func"])["runtime"].median().reset_index() # Print the overhead incurred for each task (estimated as the median of the # time difference per task). for k, grp in curve.groupby("func"): c_iter = grp.query('method == "iter"').set_index("N") c_parallel = grp.query('method == "parallel"').set_index("N") overhead_percent = (c_parallel["runtime"] / c_iter["runtime"]).median() - 1 overhead_time = (c_parallel["runtime"] - c_iter["runtime"]) / c_iter.index print( f"For func {k}, overhead_time is {overhead_time.median()/1e-6:.2f}us, " f"increasing runtime by {overhead_percent * 100:.2f}%" ) # Plot the scaling curves. fig, ax = plt.subplots() for key, grp in curve.groupby(["method", "func"]): ax.loglog( grp["N"], grp["runtime"], label=key, ls=LINE_STYLES[key[0]], color=COLORS[key[1]], ) ax.set_xlabel("# Tasks") ax.set_ylabel("Runtime per task") ax.legend( (plt.Line2D([], [], ls=ls, c="k") for ls in LINE_STYLES.values()), LINE_STYLES, bbox_to_anchor=(0.3, 1, 0.3, 0.1), loc='center', ncol=3, ) plt.colorbar( ScalarMappable(norm=LogNorm(1, 200), cmap=plt.cm.viridis), label="Matrix size" ) plt.show() joblib-1.3.2/conftest.py000066400000000000000000000055051446465525000151750ustar00rootroot00000000000000import os import logging import faulthandler import pytest from _pytest.doctest import DoctestItem from joblib.parallel import mp from joblib.backports import LooseVersion try: import lz4 except ImportError: lz4 = None try: from distributed.utils_test import loop, loop_in_thread except ImportError: loop = None loop_in_thread = None def pytest_collection_modifyitems(config, items): skip_doctests = True # We do not want to run the doctests if multiprocessing is disabled # e.g. via the JOBLIB_MULTIPROCESSING env variable if mp is not None: try: # numpy changed the str/repr formatting of numpy arrays in 1.14. # We want to run doctests only for numpy >= 1.14. import numpy as np if LooseVersion(np.__version__) >= LooseVersion('1.14'): skip_doctests = False except ImportError: pass if skip_doctests: skip_marker = pytest.mark.skip( reason='doctests are only run for numpy >= 1.14') for item in items: if isinstance(item, DoctestItem): item.add_marker(skip_marker) if lz4 is None: for item in items: if item.name == 'persistence.rst': item.add_marker(pytest.mark.skip(reason='lz4 is missing')) def pytest_configure(config): """Setup multiprocessing logging for the tests""" if mp is not None: log = mp.util.log_to_stderr(logging.DEBUG) log.handlers[0].setFormatter(logging.Formatter( '[%(levelname)s:%(processName)s:%(threadName)s] %(message)s')) # Some CI runs failed with hanging processes that were not terminated # with the timeout. To make sure we always get a proper trace, set a large # enough dump_traceback_later to kill the process with a report. faulthandler.dump_traceback_later(30 * 60, exit=True) DEFAULT_BACKEND = os.environ.get( "JOBLIB_TESTS_DEFAULT_PARALLEL_BACKEND", None ) if DEFAULT_BACKEND is not None: print( f"Setting joblib parallel default backend to {DEFAULT_BACKEND} " "from JOBLIB_TESTS_DEFAULT_PARALLEL_BACKEND environment variable" ) from joblib import parallel parallel.DEFAULT_BACKEND = DEFAULT_BACKEND def pytest_unconfigure(config): # Setup a global traceback printer callback to debug deadlocks that # would happen once pytest has completed: for instance in atexit # finalizers. At this point the stdout/stderr capture of pytest # should be disabled. Note that we cancel the global dump_traceback_later # to waiting for too long. faulthandler.cancel_dump_traceback_later() # Note that we also use a shorter timeout for the per-test callback # configured via the pytest-timeout extension. faulthandler.dump_traceback_later(60, exit=True) joblib-1.3.2/continuous_integration/000077500000000000000000000000001446465525000176025ustar00rootroot00000000000000joblib-1.3.2/continuous_integration/install.sh000077500000000000000000000044121446465525000216100ustar00rootroot00000000000000#!/bin/bash # This script is meant to be called by the "install" step defined in # .travis.yml. See http://docs.travis-ci.com/ for more details. # The behavior of the script is controlled by environment variabled defined # in the .travis.yml in the top level folder of the project. # # This script is adapted from a similar script from the scikit-learn repository. # # License: 3-clause BSD set -e create_new_conda_env() { conda update --yes conda TO_INSTALL="python=$PYTHON_VERSION pip pytest $EXTRA_CONDA_PACKAGES" conda create -n testenv --yes -c conda-forge $TO_INSTALL source activate testenv } create_new_pypy3_env() { PYPY_FOLDER="pypy3.7-v7.3.7-linux64" wget https://downloads.python.org/pypy/$PYPY_FOLDER.tar.bz2 tar xvf $PYPY_FOLDER.tar.bz2 $PYPY_FOLDER/bin/pypy3 -m venv pypy3 source pypy3/bin/activate pip install -U pip 'pytest' } if [[ "$PYTHON_VERSION" == "pypy3" ]]; then create_new_pypy3_env else create_new_conda_env fi # Install py.test timeout to fasten failure in deadlocking tests PIP_INSTALL_PACKAGES="pytest-timeout" if [ -n "$NUMPY_VERSION" ]; then # We want to ensure no memory copies are performed only when numpy is # installed. This also ensures that we don't keep a strong dependency on # memory_profiler. We also want to ensure that joblib can be used with and # without lz4 compressor package installed. PIP_INSTALL_PACKAGES="$PIP_INSTALL_PACKAGES memory_profiler" if [ "$NO_LZ4" != "true" ]; then PIP_INSTALL_PACKAGES="$PIP_INSTALL_PACKAGES lz4" fi fi if [[ "$COVERAGE" == "true" ]]; then PIP_INSTALL_PACKAGES="$PIP_INSTALL_PACKAGES coverage pytest-cov" fi if [[ "pypy3" != *"$PYTHON_VERSION"* ]]; then # threadpoolctl is only available for python 3.5+. PIP_INSTALL_PACKAGES="$PIP_INSTALL_PACKAGES threadpoolctl" fi pip install $PIP_INSTALL_PACKAGES if [[ "$NO_LZMA" == "1" ]]; then # Delete the LZMA module from the standard lib to make sure joblib has no # hard dependency on it: LZMA_PATH=`python -c "import lzma; print(lzma.__file__)"` echo "Deleting $LZMA_PATH..." rm $LZMA_PATH fi if [[ "$CYTHON" == "true" ]]; then pip install cython cd joblib/test/_openmp_test_helper python setup.py build_ext -i cd ../../.. fi pip install -v . joblib-1.3.2/continuous_integration/run_tests.sh000077500000000000000000000053451446465525000221760ustar00rootroot00000000000000#!/bin/bash set -e echo "Activating test environment:" if [[ "$PYTHON_VERSION" == "pypy3" ]]; then source pypy3/bin/activate else source activate testenv fi which python python -V python -c "import multiprocessing as mp; print('multiprocessing.cpu_count():', mp.cpu_count())" python -c "import joblib; print('joblib.cpu_count():', joblib.cpu_count())" if [[ "$SKIP_TESTS" != "true" ]]; then if [ "$COVERAGE" == "true" ]; then # Enable coverage-related options. --cov-append is needed to combine # the test run and the test-doc run coverage. export PYTEST_ADDOPTS="--cov=joblib --cov-append" fi pytest joblib -vl --timeout=120 --junitxml="${JUNITXML}" make test-doc fi if [[ "$SKLEARN_TESTS" == "true" ]]; then # Install the nightly build of scikit-learn and test against the installed # development version of joblib. # TODO: unpin pip once either https://github.com/pypa/pip/issues/10825 # accepts invalid HTML or Anaconda is fixed. conda install -y -c conda-forge cython pillow numpy scipy "pip<22" pip install --pre --extra-index https://pypi.anaconda.org/scipy-wheels-nightly/simple scikit-learn python -c "import sklearn; print('Testing scikit-learn', sklearn.__version__)" # Move to a dedicated folder to avoid being polluted by joblib specific conftest.py # and disable the doctest plugin to avoid issues with doctests in scikit-learn # docstrings that require setting print_changed_only=True temporarily. NEW_TEST_DIR=$(mktemp -d) cd $NEW_TEST_DIR pytest -vl --maxfail=5 -p no:doctest \ -k "not test_import_is_deprecated" \ -k "not test_check_memory" \ --pyargs sklearn # Justification for skipping some tests: # # test_import_is_deprecated: Don't worry about deprecated imports: this is # tested for real in upstream scikit-learn and this is not joblib's # responsibility. Let's skip this test to avoid false positives in joblib's # CI. # # test_check_memory: scikit-learn test need to be updated to avoid using # cachedir: https://github.com/scikit-learn/scikit-learn/pull/22365 fi if [[ "$SKIP_TESTS" != "true" && "$COVERAGE" == "true" ]]; then echo "Deleting empty coverage files:" # the "|| echo" is to avoid having 0 return states that terminate the # script when the find uncounters permission denied find . -name ".coverage.*" -size 0 -print -delete || echo echo "Combining .coverage.* files..." coverage combine --append || echo "Found invalid coverage files." echo "Generating XML Coverage report..." coverage xml # language agnostic report for the codecov upload script echo "XML Coverage report written in $PWD:" ls -la .coverage* ls -la coverage.xml fi joblib-1.3.2/doc/000077500000000000000000000000001446465525000135365ustar00rootroot00000000000000joblib-1.3.2/doc/Makefile000066400000000000000000000010271446465525000151760ustar00rootroot00000000000000# You can set these variables from the command lines SPHINXOPTS ?= -j $(shell nproc) -nWT --keep-going SPHINXBUILD ?= sphinx-build .PHONY: all html clean all: html # generate html documentation with warning as errors html: $(SPHINXBUILD) $(SPHINXOPTS) -b html . ./_build/html/ # remove generated sphinx gallery examples and sphinx documentation clean: rm -rf auto_examples && rm -rf generated && rm -rf _build view: @python -c "import webbrowser; webbrowser.open_new_tab('file://$(PWD)/_build/html/index.html')" show: view joblib-1.3.2/doc/__init__.py000066400000000000000000000001371446465525000156500ustar00rootroot00000000000000""" This is a phony __init__.py file, so that pytest finds the doctests in this directory. """ joblib-1.3.2/doc/_static/000077500000000000000000000000001446465525000151645ustar00rootroot00000000000000joblib-1.3.2/doc/_static/custom.css000066400000000000000000000024261446465525000172140ustar00rootroot00000000000000/* Avoid white space above the logo */ div.document { margin-top: 0px; } div.body { padding-top: 30px; } /* Main page title */ div.body h1 { text-align: center; font-size: 270%; color: #643200; } /* Secondary sections title */ div.body h2 { color: #643200; } /* Third sections title */ div.sphinxsidebar h3 { font-size: xx-large; } /* Python examples code block */ div.highlight pre { background-color: #f5f5f5; font-size: 80%; padding-left: 10px; padding-right: 10px; } /* restyle table */ div.body table.docutils tr{ background: #ccc; /* fallback if nth-child is not supported */ } div.body table.docutils tr:nth-child(odd){ background: #f8f4ee; } div.body table.docutils tr:nth-child(even){ background: #fff; } div.body table.docutils td{ border: none; } /* Move the ads down in read the docs */ div.sphinxsidebarwrapper ul:last-of-type { margin-top: -10px; margin-bottom: 100px; } /* But don't apply this to nested ul */ div.sphinxsidebarwrapper ul ul:last-of-type { margin-top: 0px; margin-bottom: 20px; } /* Sidebar subtitle */ p.caption { font-size: x-large; font-weight: 400; } div.body p.caption { color: #643200; font-size: 180%; font-family: Georgia, serif; font-weight: normal; }joblib-1.3.2/doc/_static/favicon.ico000066400000000000000000010137761446465525000173240ustar00rootroot00000000000000 (  TSVVVV6VVVV*VV@XWVV3VhV.UVVWWUWSUVWVVRVVVVVVVVWUUVV)VVVVVWVRVVOUVVVTV(VvVVVVVVVVVV VVRWVVVVVVVVVVV:V VVVVVVV@VVVVVVVVVVVVV~UVVVVVVVVVVVVVVVVV_VWWVVVVVVVVVVVVVVVVVcWUUVVUVVVVVVVVVVVVVVV1VVVVVVVVVVVVVVVVVVVJVTTVVTVqVVVVVVVVVVVVVVVRVVVVVVVVVVVVVVVVVVVV3VVXWVVVUVVVVVVVVVVVVVVVVUVVVVVVVVVVVVVVVVVVVVV VVV^`VV<VVVVVVVVVVVVVVVVVQVVVVVVVVVVVVVVVVVVVVVUVVWWVWVVVVWWVV&VVVVVVVVVVVVVVVVVVKVVV VVVVVVVVVVVVVVVVVVVV#VV WVWVUUVX`VVVVVVVVVVVVVVVVVVVVAVVV VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV7VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV.VWWVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV$VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVZVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVWVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVTVV|VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV U$IUVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVTUUUVVVViVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVUmQUU*UWVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVmRUVUWVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVmTUUUWVU!VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVmRTU$IHWVV#VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVmRUV$ISWVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVmTUU$IVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVmQUVVVVVVVVVVVVVVVVVV VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVmSVVVVVVVVVVVVVVWWVVV"VgVoViVaVUVIV<V2V)V!VVV VwVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVmV VVV V(V1V:VDVNVXVbVkVoVNWWTUVUVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV0VUWVV<VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVuV[VQVSVYVeVxVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVWUVWVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVtV5VTVVVVVVVXUVVJVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVDVVV!VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVDU WVVUUVfWVUVV_VVfVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV^VZXUViVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV@VVVkVQXTUVVVVjVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV VVV VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVkV VVVWUYVVV&VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV6VVVV9VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV:UVVUVWVVVkVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVWWVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV#VVVUVVZVMVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV-ZVVVVVDVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVFVZV9VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV#VVWVWUVMVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV}VTVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV9V$XVWVWVkVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV6VWVWVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVgVVUVVWVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV.VUXVWWVrVVVVVVVVVVVVVVVVVVVVVVVVVVVVV VUYVVV&VVVVVVVVVVVVVVVVVVVVVVVVVVVVVV VSUVVTVZVVVVVVVVVVVVVVVVVVVVVVVVVVV;VVXVVVmVVVVVVVVVVVVVVVVVVVVVVVVVVVVVYPUVVVDVVVVVVVVVVVVVVVVVVVVVVVVVUUUVVVVVVVVVVVVVVVVVVVVVVVVVVVVzU VVWVVWV,VVVVVVVVVVVVVVVVVVVVVVVV<VVWVVrVVVVVVVVVVVVVVVVVVVVVVVVcVVVQUTVVVVVVVVVVVVVVVVVVVVVVVVVVWVVV'VVVVVVVVVVVVVVVVVVVVVVVL\VVVVVV VVVVVVVVVVVVVVVVVVVVVeVQUUVVVVVVVVVVVVVVVVVVVVVV8VUU!KWVVVVVVVVVVVVVVVVVVVVVV*VWhVVgVVVVVVVVVVVVVVVVVVVV,VVUWVVVVVVVVVVVVVVVVVVVVVV VXVV5VVVVVVVVVVVVVVVVVVVVZWVVVVVVVVVVVVVVVVVVVVVUWVVVVVVVVVVVVVVVVVVVVVVZVVVVVVVVVVVVVVVVVVVVVcVbVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVIVZUMVVVVVVVVVVVVVVVVVVVWVVVVVVVVVVVVVVVVVVVVV9V;WVVVVVVVVVVVVVVVVVVVW jz|zwtpmjW VVVVVVVVVVVVVVVVVVVVV2V KVVVVVVVVVVVVVVVVVVVVW;VVVVVVVVVVVVVVVVVVVVV4VAVVVVVVVVVVVVVVVVVVVVV/VVVVVVVVVVVVVVVVVVVVV>V6VVVVVVVVVVVVVVVVVVVVV]tVVVVVVVVVVVVVVVVVVVVVQVWT5VVVVVVVVVVVVVVVVVVVT] <VVVVVVVVVVVVVVVVVVVVVkVWVVVVVVVVVVVVVVVVVVVVVWW5YVVVVVVVVVVVVVVVVVVVVVTVVVVVVVVVVVVVVVVVVVVVVSUXHtyWVVV!VVVVVVVVVVVVVVVVVVVV VUVV4VVVVVVVVVVVVVVVVVVVVVXV(sa >|LUWVVVVVVVVVVVVVVVVVVVVVVV'VWUVVgVVVVVVVVVVVVVVVVVVVVV(VVV<I9it(VVVVVVVVVVVVVVVVVVVVVVVVVV\VUVVVVVVVVVVVVVVVVVVVVVVVVV=VVVI2~QC WVVV.VVVVVVVVVVVVVVVVVVVVVVVVVUUVV*VVVVVVVVVVVVVVVVVVVVVVVVVWVVVT F<]WVUVAVVVVVVVVVVVVVVVVVVVVVVVVV-VVUSWVwVVVVVVVVVVVVVVVVVVVVVVVVVqVVVRl?)RQVVWVWVVVVVVVVVVVVVVVVVVVVVVVVVVWUUVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVWWg^VVVVnVVVVVVVVVVVVVVVVVVVVVVVVVVVV*VVWUWVuVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVQT^w-g7 ZVV VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVUVVWVV,VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV)WUVT^ =`nH) SVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVaVUSVVV VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV7UUSJP6_o@ IVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVBWUXUVXVrVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV=WS?>VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV5VYYUUXVSVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVkVT54VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV2UXWWVWVGVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV&VT-,VHVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV:UVVYVVVQVPVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVXW%$VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVNRVVUVVVVVkVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVEVVUVtVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVtVVVWUTKVVV$VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV)VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVAVVVVVUUVWVVeVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVViVQVUVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVFV VVVV*NUWVVVXVV`VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVU+WVVHVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVmV/VVWVVVVVVUUVVCVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV[U:UVV VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV~V{V|V~VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV5VUT$VVPVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVXWVVVVVVVVVVVVWW*XVWWV*V9V6V/V&VVVV VYUVV VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVXN B"huWVVV!V)V1V9V@VCVV? g) A0VVVVVVVVVVVVWVVVV VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVXZ)eIJVVVVVVV42^]XVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVXe / g#eiUVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVXl 4'Qnq[UVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVXm :Z"wsWWVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVXn "fonSUVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVWk $aXVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVWg%GUVVVV~VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVIVVVV,VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVp5)_VV6VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVOVVAVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVW0oVVNVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV VVZVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVW[5x_VVgVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV V"%aVVsVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV7XUVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV'WTVVVVVVVVVVVVVVVVVVVV2XVUV W VVVV VVWnMVaVVVVVVVVVVVVVVVVVVVV7VUVVVVVVVVVVVVVVVVVVVHVUVUVWVVVVVVWVUVrVVVVVVVVVVVVVVVVVVV1VVVVVVVVVVVVVVVVVVVVb[UVVVWVVVVVVVVVVVVVVVVVV)V=VVVVVVVVVVVVVVVVVVV}VVUVVVVVVVVVVVVVVVVVVVV1Vv/VUVVVVVVVVVVVVVVVVV VWVVVVVVVVVVVVVVVVVVV5VI UUVVVVVVVVVVVVVVVVVVUVVV-VVVVVVVVVVVVVVV$V#dUWVCVVVVVVVVVVVVVV#VWUVWVVBVVVVVVVVVVVVwV%T\ 1VUVV VFVVVVVVVVVVV2V\ +VVVVYVVVVVVVVVOVVVThUVVVYV$VkVVVVVVV>VUSVUTVnVVVVVkV(VVVYX1rVWVVV V7VVVVCVUVUVVVdVvV5V VVVU]9WZVVVVVQVUVVWWVVTUmWVVVXWVTTU/k od\ZZZ^dlx+Y[. 7[CEX yV"mzy>V,+2<GR^juh3= O@P`/^CG 9u jaYL H:<RB16D1?P;A5eM8&n) -k^L{ SfA,#cuj(M2c^\ji9_:8;^ qs33+ H-lgWNF1*P :q/ 3w l. 2sm- 1000000001( +k[#-m-&\",lNDc%-kys5`,! rWUN1& gq` B $T@o DI6!j ?F@ ^Ta5 &'  Yh?@&2$'5-9F+KZ,Wi0VN9FH-vaK-s|(29QnGLayG ^+h^"o<AHd|ume^BA. %+28@JSL 2[>- +>XtyLI#YQ$T?n :U#hx]tTo Ni EJ#W5m<xtA~*}4D{Cw K Hz>s!O Kt9m` Hk-`,V CT _!B?p 7}} j3>e~N#jG(KwW3s Z 9\`@' w-x  -fYM*OT!c8{/7,%?OY1B$$&]"AY!???0???????????|@<<?<||<p??> ?p??????????????????@?? ? ??joblib-1.3.2/doc/_static/joblib_logo.svg000066400000000000000000000363521446465525000201770ustar00rootroot00000000000000 image/svg+xml joblib-1.3.2/doc/_static/joblib_logo_examples.png000066400000000000000000000660671446465525000220700ustar00rootroot00000000000000PNG  IHDR!m o pHYs+tIME*;RCbKGDkIDATx݅wX7v03344i&CԫNIlKYZwyyO5L>|0͉KMM[vvvO*++V__֮ dlll/לp &dA.mmmp$[2>s@yP(_r2h _p2h _L&˙XY?(|XY?(|XY?(qd,РX=(|O@>Ǐ 2hPw  4% _Cd,Д>(|@Sbhq!@5`𰧧'??n]  _228??MIIyxxxPƒBe@d,<==>--NǏ XFt282 B7kmA10$6Xidd$===)?99Yw?(IUUfMu 2qqqVUG4XϤ.\ l6aJMM~u8mxxXr!tD0 ))I{*Agg'C429994 X+.٬}_b~L&d,aPTVٻ+++F1-!!F2 )*uuu(c+&%%) d,aUZHHHP~NB32RŰt8 9i4 mrrRQ 33YgbFE=99QV^^J @w)ýnlqqR!ODSٻ ١  (\(@uaB:??_^^sMMMFFRW~~~SS@*ˏ0Q lJ%L 'Q? Į*<}|Ć򿵷 ֿl0&R5.Gq2(aϧQ lwwwʿf39I7@+**8PSRRB @v*w.333?~LMM% ɹA]]K`QJ棐#i cS^YYyyy պ=00P^^@lnn"|4oookjjh c2IaaaOOPwwwuuuZZFL&SEEȷo~ytt;X/ cDJJ !r>Rlkkk\\fX"I{{;]d,())a.2H0200@ @$G?K >}bdvv.2H 0L @$ǙG?ڢK ̣%@uQO c('777t DyB @$ΘG?q\t Dy@ @ @$G2LDb0B @S)[YYYD2d,,0I @ S)[ O  ]d,d,@d===t D ׍" akk+##Ix]MMfcm̝;%$$,..:@Gn`00k᪫{zz_BjJJ 3%t ,++cDsttDc 5ӧ8E@\N&C;Phyy9))HrrʊPk42 [ yޑ`0y<Z 631) @S\.WWWs OEEEMMMjNNjj*sIIIkkk~cc)HE5?d^XXzNgkk+S@BpX,n{jj*--?8!!𐉀orr299yxxp(fv@fpv e2 `0fЛrBMMBVh F___||pUɼ@ XFj>;;bT$111+LW.;ղ (T hjj ] {ڑ>mpM{W%t90MHƈۥuyww'_{{{[XX,Zjjl{hY,?Yd, l^&)$KP(h2mt%2X{y< qfccIjVVV2'+Ex||0 ",2..TYWW7$f#@$466&r)nGUUw $cJ X%%%RβhN᪱Iм(wv˿`Љek•IM^^sɻ vb VL+++ {!!-w6 !댌fH .**a,))|-hi6440U\\懙F144$GiiaZ!c?x^"Ljjpۛ?> i}}LH61#XO~,WB=;;{RJٽ?:;;1BW[555I4HhmhIIIjj*3 _Ikjj*99Ya}}=SB z+I矎&cAommG1& }2m"cAw!g2}; в]6A ׯd,@dn&hPAA=bhB>U[[K4hee1A4]4}Đ@LLNNm!cAGNd̄+t`2e4;Y__g2`0@6B׵X,d,@iy .}N7d,_yVd,@/i| v5d,-@Xd,@_i =O4d,$u}}Mill,CƂ~@"d,@ Z2tV;==e!cA &2"*  (,, B,d,]"b{{iOqq1 SX|QZ]]eB!c @ B~E3ϧK@RRR|>S  %D`jjIQ0ΦW@X= x<222AzNF郌˗/4Mxv&2.6M,o{|WKK }TRR”AW=2X@~? MV)'// s\d, XoZYYa cau x㸸8Z'  xHKKi;___3}ׄB:&%;;oBLOO+ ---L"d,.// "TB~xrrrh1twwDŽBQz)d,@L+d,L[1w~"JKKs:L1d,ڗ/_ :r cAfHdssΦ DV+ НO>@R555P>dDp@&၌Bcc#]dSQQ'd, [Xll6[,2SBȪĐܤ@ qiii48F41W__O A]BPg2zt4P2nvJG*@222^/ P)(0 Pj6iaXF 嫨vXР}:: PϗEۂ좿UVVV'_奥d% ~ET`hh0 999?8==X,"փ`0t:~:;;+|VZZ:dd,@F# b2'&&sR~_H`{{{,$8~2(RD cVUUĒ6??uuD|aaA2\)#33{ggtjԔO iow2q||>^Ԟ>Zuuu5>>^XX9o N0 rBBmH>111999+++77QSwwP(Z CnssS-!l0 F1K'H?::󗖖pzzJƂB1FQkkkn [cccyyy HGv8 I3+4===::*\iwuulq򲳉pe&+\K* "`SuhIiiоaO΄d21T ! 9PIIII*~||LƂXV@_.k=n" d,(0 s;::KP5`XPaZJII8F }}}OOOr$CXP 3;;Z]]e]}Ēvmnn&iAX9JU]]}ssC*0iqO 7sXM(!(SaaEd.%[[[#c!f9"Ҥ...5ɸ3MMM!#..P!ꊌ|ʑ{vvFafkoog$CQXẓYppPթ-!( qQ 0'c!<`%PBQnCQmr( ۣs}}}~zԤ뢢"9bY544Px]*Qhd#YNBV+C6Jԉ|=bRr1G$$$x^2$'Log6y>g_~MJJ+++d,Hnggb(@*DYY E_ ᇇ) /̓.%..J>,l: >2$411AA6!5ж,IJJ{d, oPC6&i+Nguu5lmm k ]/StxS0d 2$A#<23382#C0s8d,/'' v;pL&*R[XX cA+E r"svv&Z]] zO!iiiTJƂT" # X,PPPv2°Ĵׅ@ʂXGqq1q=??G0 XV||<%utt;DrYQnnndHQ&`AUXXHAd,D!;;CdzzzBuIcVQQhG' "=- | av#`Xeeeʐ9[WWG)b1@ @ IAq̤~]]]b%cA%^,CJJ 5q쑱 ŮW Xo߾MCh4;XlPnbׯl?IJJ:99}ԑ thkkc,("%̼bȑ 2ŒG_"S9PA ?+))xw2bR:W]]Ǟ4PH'^466z^ &v X%\@nwNNEϟ?K >eggG @R n0Xh#99ၑ١Zd23X{:C_^^2 _|f&%%EEƂ1>z.!`0XSSCGffwXSvv6EasHMMx 77j}bB<OFFŮjIIIPX,tՉ666ww2bfkk: ]A(*//UmyyKbbbl?Lga TVVF P6޴PꕗGƂSVЅXVD߄]^^R222¸}W}2tgaa&|'99&! A0599IPZ2t'??W V@b) iX|LEй ZJ=<<#'''ZFƠy^ne: :255E٫_'^ԨinnU!...EaUJ#YYY*i cll:@]p8ԼZ\^^2b8N}\*0\_tvvTgaa]h8>>f9Xc %%%U*++iR\\LƂ.RʷX~kgg.&) q>ޕ/99Yo 5/G# wqqA+cx4B]vvvXи%J], cx4 'cAz{{)u#xP) imm%cA)u;::bob建eP(11RW\gbo 4 0QX,\;4Tn#6 `tssCP-24k||"WzF)BZZY|EG666@X(KEXЬ\L&aayzz{E~~> زOW&d, W}F)QZߓA۔b%''˿s + \AV]]] Q bYYYUd,ȤVm(ϟ?FXxJBQd0\.C)D X^*T2(`0D3Q>a&d,h _PQ(%C Ia+9CDX-j [߈ DbOT-DJZ>~pDMKQ!y>FƂ/55JVI- ZZ$$$S ܵT(X{ZGGG- cVp8ɀ(*z>>>]+### % %&&XT}[rmm]+D Ƣ[[[QnXJwkllƢ%/"ch_ u@D,{תԞ D+}'s|Eok&..Noz`Z-:ֶX Z_(@j4i/-2ъhk :C'lVxLEf/ccc~" W/|kd,2tjnnDM{`||EƂNP膇i/XЩs*,!mXЩ;*[ŗ/_Xd,f"mll^Xd,"^ 'cG! Q0XУ$9n2 zI9R@E"cA )@ t:;;Xd,Qyy9HA[[=2J"cAjjj(@ tXd,Q]]HǏXd,QCCH6K"cA)@ wwwt<. cGMMM {: >알EƂP ҩ"cGmmm : d,2rHE h0XY!2DVTTD"cAXHXxQXXH"cAػcE~~>=bR@"|/rssXd,Q}}=HAvv6=)|Aff&=)a HOO'cG : d,2rpttD 99EƂP0$&&XУBL"cA)@ ;;;tXd,QJJ Ha{{` cSSS.(@ tŕt' t.PVVVh/:V` c:l~)))!A0Z ?fy0;[ɔ~2ѱUowfggVd,V[[[BBţynjy@tG+2^wwwe6)'dh;Zw2^L&1B`knnh,ZUVV.,,̈́d,F(JHH())IoJ\C"NAƂ|***(reNLMMFOXЬ>\@JKKi#CƂf-//Sʔ"c!@Ƃf %g(zqqqԹ2-z@KXiN-ɡy\hbQd,ȪRW[( Z*Ցqcccb0DZ q;;;beggB!F)N cA Y;B#cA}Q G0LOOiB|||LFƂ(x2^Q 舎1$d,ȭW5F)fwXPU ^l*:Fvd,p Tl.'''d,B0dٻuww3PWXff&BE\. zQUUE+dUKooo."YYY*d,/_({/*@PV2td{{Wl'(抑EZ Q .߾}#cA_Ҩ|ծ}b\.LsPX4B^xi||.%%%10d,ЪGUKII-K__ sxxH,,,Tgccx< W  fggT=ϧU ܤNJJJ("cAhTq+77V@"cA5VVVhjqyyɈnhzzEee%#hjGXP(@P}-t Ud2 *))E~~>_0|t_b>~Xv0h+SSSJ k4IKKx<[HJJUjkk]c|q ၌]3@]F: +Ejd,lNmm- )vUB) kgg1 [__5/[[[ԿznF24fddP暹XQ^z{{Ф \KbB4 lqqqgggjhՕ`5fYNRגخuD&cJ[233XР\\{2 ZONNXД[O OOOЀHQkollۼBmYYY|`j )g!cAN0A U ~1RKC?~z,2xjkk)`]ILLX, ~̤pOOOyyyUTTpWss3ūg999d,oxPd[I biiERRD;* mDY\\Ue0vwwXPa~P!\>==QP8êr@Ƃ^(((z+ 555Q/qt c!*T@YToBkO2"g255E@~Jyu.gSĄ(͍l<񦼼#c!р7L&aP/PәEaҮXxZT bRKBB) 22ޯ@ @@]]ň U6B˥*'P2D#= ﵼLBu Rh+|q86YFloh`J[0-a{mmmT#b(55uggJԆgZ b.777 {ggg$b>] lBB1 &c!*$bd23^"?e (((8>>*񱱱q w~(0m?< -l[[[nn. 2LDƂ6 G 'P\\#@ rp|699U]]C@V}}g"24O*--윘orDbQoX,8588(sKHOOSP&,їXidP!'&&fffUVVVYEESXX$?'h 2@Q  Psr |顩B d!cAnwzz:} b.33S?_M cAhmszXЋP[[2Nh͹sPәJHJJz~~&c!<ս3Ituu@X=];f:T4 =@{w>貋xެ,zH*33vYi}x߄rvEy}!Cc'Xqusc1{K꧕Fs}ZƄX֕Z Ԡ9ވ0/UUUMxqm X?bA˴B^Lȟ&Nx]VV%$c;cMw*3cYS}: ٩ῇ0fVVVRdq(5cX۳O7j9ޓ-wͿ%^rRdgAf*:?BQN XEՆ'noo,E3֏C~ϡvjͮchmm%Kh{FK8ӝCAevj(S I w'cgYK8\J|!6<~T  ^]]7SdWjo"c؀ M$KXA'7kK:7@l@)c}SKTAKVq:sXXZZ"QZԒ^Bul[j9i"nb<jKLke,$1 Vbl666UdGue>1qnG|7'hͳޙsmN\+CΥp|WQƸpc_߽_R^?]Ar{㡡,~ [_HcG j.W_$o&igg\EԘ8]lS +** BD+g,% ù C뻵3 õޞxUTTXm% inbNOOIWzXj27@~+g,U|q#ons b₀ h]s$'Ǐ,]g,|/ADÇ]WQtCW\\-KK-6Ǧ&!ߌޚ"ho;bVRzxx cgw ێ. n2^3:AD=h0h hXz=%h9:;; ϟ?wsBaz׷Z$e2KwWdϣolCt'`$hpLwx !!v%vEz8_z I2~V~wi6iy Gkk:qD~!l~~/OyMo&%%@f`WJ=/>;;K%cK qD7o9YYY92<5G4ysooo X:ٻi+#h؟ X=zK"X<(%ci3 oBVA\\=bǏd,-l-.bVWf &m6KB!֔9)?`u}e__df0yP*p;_vphllu6I+9&Z <9ǃ竪f]kY[QQA566+:XA>XEశgn~nҩJ+EBŬL#r@ %%%绎i6cb977Wi`rrjff&--N cccBzE~~¼<]ߴG[W>w1+㗘e|\\\cc/7{x&/r(rNNn{X#w8ފYzhhX222b؏L&S__ӟF#B(..SxxxW/ɲר^2VD`mЛRSSoF _|y~~~{ Phii)11i!!9MLL7{n01;333Vҕ3Va 8"YwwwrƬ_njVVV2RZZ*4z,==d,5oz XQ,;aRȆz(5L< ]Dڅ>4NIYB̒n Ѓ}>_c*//аۅVzztjvv6n!V̲! =Nܽ6887MͥꪐۤȂ,rf z.o,{666ql_ ]."^F -=o퐱'-BbVrJJJi=U r]]]L4559gPHqQ%!_KƊE -b֧ᗘUZZA)++䬈d(@eIOOO#tf`1E,u_֊KUẎksBynG_Nrλ]c/Bq,9ݛC?Vzl Vyy=Ƀ/Bot<[_KXcS!\kBDҒG&? L(v~%`qn|fzQ>.<~Dyď4l{iTz9%UMzzT,))iyyY /BYKC2|]h!T0V[Ӟ%ɖZ[ҙ9b t!qCHN-C}%zK}&''ѵ2oil}L 9g_EV3N88ww7B夤$&B@t 333[3V(d\eyyf̧ߎ~m%\ƪ]N'ߓU__6x#c.ڳG\+CMMiEΠoGlζB7j05Q'[%gkm-\ݹ{smɔuſD|###"/TH]+`c2SBroM ElGIe/=kDuII %G"ޛ\_b@й;TN{{[oҙ+f?~`pbbYx833B+'67zK<{A[ѣ&]8f:6xCx]bb"[3 9\exi]˷J gkeڞM%qA#vƲV1Dˏݛ$-ׯ̠VVV/c/d{ t\cֶL~Vlt?3(mbX^f/ oϵ7gʴ%=KS3;88 F dNWw{wk:~k֔D?3(SbX-|bu1BA2$DbX/cBR-IOy# hחJۭ3;::"Fw{ƌ%/5U{Zx wkW[^^f^G{3sKK@@,`{V39Rv?Ɒ ތ{Y=#k_S*;|pmTkjjx:XޓM*Q(ッ7ᚵ5! ~q!fPuAf籏Up+ ؟^&d+žc nnn3VacH egyU]1ޝggo 7NP-F#`jhh`^700@3w~db4LcM V TVV2ꢓ㍌x~b~{of#[8b4̠ZZZhx#c2?E )D[~! 72CF{Ca!-DIII̠hx-c]ϼQUa^7Y_t#/b0AӽZ3-EȽ\v[w!*o7E[x^OMnӝLKU'cO7|=:ڕYA'xSBB}e,[WRd;=N|<2cEQ'&@ki))/̧rޜdEBU?'~?{"+q=4091f2+MI.kSMuYYYMMML횛'&&T;<36(rv1E{ Mƍ))v؟!wX:TIp{<)!ueddbokkn2Ԣ-p- eB!/m-2hdZuۻ󯨑5e:#"6" .;(&q,iR~NÜzTޚgcQb54|MEEȄNaO&SSS͏?_8_KKK[[[ ,Y<'{U~N~ֿ}UT*,yOhalbrʧON\jY]]]ϟ?/((8w秢X,ҒmOYwaag5| 0E~odF6pD.]Z]] nJ"{_ӏ3nIII}}/N-]AcdGa{,K~* sCٻ捍/ܖL&溺*++ݻwŜMTϟ{_ʜ`yyY>6WP8f|~yK[8.WO)7ÿwzkkk{zzyy;;;~kmm-++rJCյkמ,O|Gzbju!f[~~>NCyyyvԔgdF3c&p*CL~B9$NN ?&W:I_A>cչ >817E8|BhwC^TqsQQȝ~@Q^^^7"ݦɶ]d'c nT}:bQ}CO H ^'o>x@Ẋ}!_BHv̥hB:+nkk wxʊU'Ejnn`gܹs~-0c,ˈBXs<ǎ)꫻%E^ZZzaX޿/{&I)^nnnFiZV7^rQUHWl3јe^9Q:<mGҝ`[d [[[/_ݽ{߲8f"ptfVg# aw5ϗͪLeeedt``@re.\PUUe,$'Kٺd}"p}̣())OkҢ(H 3VOLop7h.t+2~vXt-+\{+QhBdĄ kddD"c {Gj0&Qڧam_~E!@ 0c52 Tex+0D,>G5v9+MMM ciqjg[d|W.m_E!@ (c 52Y Tu92VMnV 1c JӠmjM5Vrr / 2V@kUCN8]VZ92;ܬ8Ε+WB X.\E!5"Mr4 kj*#cf+&OsHjT }n!5^d 2h!TU_QQ/ 2is !eXX %,zzzh= N;Yi΄^d5i2V3aLe3憩;drX[AFҔ49 XeYt_- #}R=Ɏ XAe,!%Ro_pPƚeS* 8NGF9cŹ_@ (c 2!Md,g.IYC7qu<ڼR/B%;bRg:6XGup~N,O/3NeE5KC/$P cts2ԗw]1322byXWxHqKk*N:\:l<,v(yCLV֗hdcoշ6!t}$3၆뀹8!יqx}3B}52JFT3^m 5S1ghJM _,l 2S1}JќK?8@FDWþsdo1͊Ά1;HB+34=5M9{[Jv&bg,ϣ 17Ddkّ{:[(5cy^]ѻ2[<76N ׭&cn9ِ' .fݠ @ČR8'XK pZb:j&XrV)!1c.GOؘt2:=goS,"~mJ:UR?Յ)U7@Ќe2Wx#f<`)Ӿ(lyJ!sՉz: V K'N&z(w։Atk:Bg,*3iv{SROqkg,VW̤u9Y?f(@:KEKͤH{RwñA(ed,'h;v^L櫌lyJmnz92/+g6՚""A20?sـ+<@Y&|ްZ_ sj +B&xL˗}x/`cN1:\Rbs0,cYaYYЪ)2'rZCxi2־(+c[D.g{<@ŒoL3et!bz\S*3Xݧ-HR`2zxne蓽JMv{ dXLfg*3nF5-WOll']nw'3GDęz6>R:Fz !t\{rg,@g3o,Pb.!fxe~|ϑ8 8U2憙:'6n~lSl[ W }g= ݭ5B_'Ό[=投cZ31NrWD{5a{gđ˳-'+Ҕ>ѓcJyf@t2/M%X]]fM߭OK'{Ã1ԜxI;)cq|!o~3dd @ ^&H,`dNp+B+`:bD+X2V1˶9 `Xs-Ck|L\`dc,+蘕ѴbM2V1KOio8А x2u `X,30 `XA,J?'R0X2V1. 6XY>I` cϘދ3Xf/( X @{g])"s 2V:,X;WK `@ 1=.xdYkJM 2VmG+& c5u51+"Cyqd,Q$H'P2D WOQ2Ww&CIXB( *% cb׈&<% cϳ-rI8@ %Q j+|K.'zUXw7%QV~2ks\W cZ#DiUX3'%:_5>cn\5§OK4{d,2x'$JCɯ@ _f\qusaX!KדK"6\S c,PVaXagqBIĆlBl}F(ذ?B5>&DlXaXag sy62坨*P*o}k%_G,Vu:+R_G^ȗ)c-Bzl&[j3젹4eۛKκu3Io^`=*%2fd3VD5lJt6&ϱ%Z =:Hq2d0FF#$$huN4)VslAR6jRXjo袩P Y@ .W~Yby.= cXk~#"Z c#rA^fI5BXs6 nnm\Čw~2V̥)Awmwе ?b~2V{aJ<˔:=X=~2VܸiUho(/킌 cfb}HnZJ@ 5cM u䳵2#̌v?l+LT@G>O}5=/RX-OaXfIQ2V.sd c&zYw'5ҡN4X=l*vM=JW5CBsi6&2V*ꖓ܍ޅ6ϼdszï@ kp(gʼnVfF"Ud(w}JV_ƣ}I]j+g @ƒIꓽ꫻Y0y W2ZMgmW9q!'+ Q3F N ec5<}7(3$(H+Z)2Aa霋ڎRuչ}ץd(ҽ/Oܕ@騟r}ZU73ĘIFrkĮYrYs{v xcnPc2V joblib-1.3.2/doc/_templates/layout.html000066400000000000000000000007371446465525000201050ustar00rootroot00000000000000{% extends '!layout.html' %} {%- if pagename == 'index' %} {% set title = 'Joblib: running Python functions as pipeline jobs' %} {%- endif %} {%- block sidebarsourcelink %} {% endblock %} {%- block sidebarsearch %}
{{ super() }}

Mailing list

joblib@librelist.com

Send an email to subscribe

{% endblock %} joblib-1.3.2/doc/_templates/navigation.html000066400000000000000000000005031446465525000207160ustar00rootroot00000000000000

{{ _('Navigation') }}

{{ toctree(includehidden=theme_sidebar_includehidden, collapse=theme_sidebar_collapse) }} {% if theme_extra_nav_links %}
    {% for text, uri in theme_extra_nav_links.items() %}
  • {{ text }}
    • {% endfor %}
{% endif %} joblib-1.3.2/doc/conf.py000066400000000000000000000173121446465525000150410ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # joblib documentation build configuration file, created by # sphinx-quickstart on Thu Oct 23 16:36:51 2008. # # This file is execfile()d with the current directory set to its # containing dir. # # The contents of this file are pickled, so don't put values in the # namespace that aren't pickleable (module imports are okay, # they're removed automatically). # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os import joblib # If your extensions are in another directory, add it here. If the directory # is relative to the documentation root, use os.path.abspath to make it # absolute, like shown here. # sys.path.append(os.path.abspath('.')) # General configuration # --------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.imgmath', 'numpydoc', 'sphinx.ext.autosummary', 'sphinx.ext.coverage', 'sphinx.ext.intersphinx', 'sphinx_gallery.gen_gallery'] autosummary_generate = True # intersphinx configuration intersphinx_mapping = { 'python': ('https://docs.python.org/{.major}'.format( sys.version_info), None), 'numpy': ('https://numpy.org/doc/stable/', None), 'scipy': ('https://docs.scipy.org/doc/scipy', None), 'distributed': ('https://distributed.dask.org/en/latest/', None), } # sphinx-gallery configuration sphinx_gallery_conf = { 'default_thumb_file': '_static/joblib_logo_examples.png', 'doc_module': 'joblib', 'filename_pattern': '', 'ignore_pattern': 'utils.py', 'backreferences_dir': os.path.join('generated'), 'reference_url': { 'joblib': None} } # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. # source_encoding = 'utf-8' # The master toctree document. master_doc = 'index' # General information about the project. project = 'joblib' copyright = '2008-2021, Joblib developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = joblib.__version__ # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. # unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = [] # The reST default role (used for this markup: `text`) to use for all # documents. # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # Avoid '+DOCTEST...' comments in the docs trim_doctest_flags = True # Options for HTML output # ----------------------- # The style sheet to use for HTML and HTML Help pages. A file of that name # must exist either in Sphinx' static/ path, or in one of the custom paths # given in html_static_path. # html_style = 'default.css' # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. html_favicon = '_static/favicon.ico' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. # html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. html_sidebars = { '**': [ 'about.html', 'navigation.html', ] } # Additional templates that should be rendered to pages, maps page names to # template names. # html_additional_pages = {} # If false, no module index is generated. # html_use_modindex = True # If false, no index is generated. # html_use_index = True # If true, the index is split into individual pages for each letter. # html_split_index = False # If true, the reST sources are included in the HTML build as _sources/. # html_copy_source = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = 'joblibdoc' # Options for LaTeX output # ------------------------ # The paper size ('letter' or 'a4'). # latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). # latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, # document class [howto/manual]). latex_documents = [ ('index', 'joblib.tex', 'joblib Documentation', 'Gael Varoquaux', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # latex_use_parts = False # Additional stuff for the LaTeX preamble. # latex_preamble = '' # Documents to append as an appendix to all manuals. # latex_appendices = [] # If false, no module index is generated. # latex_use_modindex = True html_theme = 'alabaster' html_theme_options = { 'logo': 'joblib_logo.svg', 'github_repo': 'joblib/joblib', 'github_button': 'true', 'link': '#aa560c', 'show_powered_by': 'false', # "relbarbgcolor": "#333", # "sidebarlinkcolor": "#e15617", # "sidebarbgcolor": "#000", # "sidebartextcolor": "#333", # "footerbgcolor": "#111", # "linkcolor": "#aa560c", # "headtextcolor": "#643200", # "codebgcolor": "#f5efe7", } ############################################################################## # Hack to copy the CHANGES.rst file import shutil try: shutil.copyfile('../CHANGES.rst', 'CHANGES.rst') shutil.copyfile('../README.rst', 'README.rst') except IOError: pass # This fails during the testing, as the code is ran in a different # directory numpydoc_show_class_members = False suppress_warnings = ['image.nonlocal_uri'] joblib-1.3.2/doc/conftest.py000066400000000000000000000011601446465525000157330ustar00rootroot00000000000000import faulthandler from joblib.parallel import mp from joblib.test.common import np from joblib.testing import skipif, fixture @fixture(scope='module') @skipif(np is None or mp is None, 'Numpy or Multiprocessing not available') def parallel_numpy_fixture(request): """Fixture to skip memmapping test if numpy or multiprocessing is not installed""" def setup(module): faulthandler.dump_traceback_later(timeout=300, exit=True) def teardown(): faulthandler.cancel_dump_traceback_later() request.addfinalizer(teardown) return parallel_numpy_fixture return setup joblib-1.3.2/doc/developing.rst000066400000000000000000000001411446465525000164200ustar00rootroot00000000000000 =============== Development =============== .. include:: README.rst .. include:: CHANGES.rst joblib-1.3.2/doc/index.rst000066400000000000000000000023641446465525000154040ustar00rootroot00000000000000.. raw:: html Joblib: running Python functions as pipeline jobs ================================================= Introduction ------------ .. automodule:: joblib .. toctree:: :maxdepth: 2 :caption: User manual why.rst installing.rst memory.rst parallel.rst persistence.rst auto_examples/index developing.rst .. currentmodule:: joblib Module reference ---------------- .. autosummary:: :toctree: generated/ :template: class.rst :caption: Module reference Memory Parallel parallel_config .. autosummary:: :toctree: generated/ :template: function.rst dump load hash register_compressor Deprecated functionalities -------------------------- .. autosummary:: :toctree: generated/ :template: class.rst :caption: Deprecated functionalities parallel_backendjoblib-1.3.2/doc/installing.rst000066400000000000000000000034611446465525000164400ustar00rootroot00000000000000Installing joblib =================== Using `pip` ------------ You can use `pip` to install joblib: * For installing for all users, you need to run:: pip install joblib You may need to run the above command as administrator On a unix environment, it is better to install outside of the hierarchy managed by the system:: pip install --prefix /usr/local joblib * Installing only for a specific user is easy if you use Python 2.7 or above:: pip install --user joblib Using distributions -------------------- Joblib is packaged for several linux distribution: archlinux, debian, ubuntu, altlinux, and fedora. For minimum administration overhead, using the package manager is the recommended installation strategy on these systems. The manual way --------------- To install joblib first download the latest tarball (follow the link on the bottom of http://pypi.python.org/pypi/joblib) and expand it. Installing in a local environment .................................. If you don't need to install for all users, we strongly suggest that you create a local environment and install `joblib` in it. One of the pros of this method is that you never have to become administrator, and thus all the changes are local to your account and easy to clean up. Simply move to the directory created by expanding the `joblib` tarball and run the following command:: python setup.py install --user Installing for all users ........................ If you have administrator rights and want to install for all users, all you need to do is to go in directory created by expanding the `joblib` tarball and run the following line:: python setup.py install If you are under Unix, we suggest that you install in '/usr/local' in order not to interfere with your system:: python setup.py install --prefix /usr/local joblib-1.3.2/doc/memory.rst000066400000000000000000000411701446465525000156030ustar00rootroot00000000000000.. For doctests: >>> from joblib.testing import warnings_to_stdout >>> warnings_to_stdout() .. _memory: =========================================== On demand recomputing: the `Memory` class =========================================== .. currentmodule:: joblib.memory Use case -------- The :class:`~joblib.Memory` class defines a context for lazy evaluation of function, by putting the results in a store, by default using a disk, and not re-running the function twice for the same arguments. It works by explicitly saving the output to a file and it is designed to work with non-hashable and potentially large input and output data types such as numpy arrays. A simple example: ~~~~~~~~~~~~~~~~~ First, define the cache directory:: >>> cachedir = 'your_cache_location_directory' Then, instantiate a memory context that uses this cache directory:: >>> from joblib import Memory >>> memory = Memory(cachedir, verbose=0) After these initial steps, just decorate a function to cache its output in this context:: >>> @memory.cache ... def f(x): ... print('Running f(%s)' % x) ... return x Calling this function twice with the same argument does not execute it the second time, the output is just reloaded from a pickle file in the cache directory:: >>> print(f(1)) Running f(1) 1 >>> print(f(1)) 1 However, calling the function with a different parameter executes it and recomputes the output:: >>> print(f(2)) Running f(2) 2 Comparison with `memoize` ~~~~~~~~~~~~~~~~~~~~~~~~~ The `memoize` decorator (http://code.activestate.com/recipes/52201/) caches in memory all the inputs and outputs of a function call. It can thus avoid running twice the same function, with a very small overhead. However, it compares input objects with those in cache on each call. As a result, for big objects there is a huge overhead. Moreover this approach does not work with numpy arrays, or other objects subject to non-significant fluctuations. Finally, using `memoize` with large objects will consume all the memory, where with `Memory`, objects are persisted to disk, using a persister optimized for speed and memory usage (:func:`joblib.dump`). In short, `memoize` is best suited for functions with "small" input and output objects, whereas `Memory` is best suited for functions with complex input and output objects, and aggressive persistence to disk. Using with `numpy` ------------------ The original motivation behind the `Memory` context was to have a memoize-like pattern on numpy arrays. `Memory` uses fast cryptographic hashing of the input arguments to check if they have been computed. An example ~~~~~~~~~~ Define two functions: the first with a number as an argument, outputting an array, used by the second one. Both functions are decorated with :meth:`Memory.cache `:: >>> import numpy as np >>> @memory.cache ... def g(x): ... print('A long-running calculation, with parameter %s' % x) ... return np.hamming(x) >>> @memory.cache ... def h(x): ... print('A second long-running calculation, using g(x)') ... return np.vander(x) If the function `h` is called with the array created by the same call to `g`, `h` is not re-run:: >>> a = g(3) A long-running calculation, with parameter 3 >>> a array([0.08, 1. , 0.08]) >>> g(3) array([0.08, 1. , 0.08]) >>> b = h(a) A second long-running calculation, using g(x) >>> b2 = h(a) >>> b2 array([[0.0064, 0.08 , 1. ], [1. , 1. , 1. ], [0.0064, 0.08 , 1. ]]) >>> np.allclose(b, b2) True Using memmapping ~~~~~~~~~~~~~~~~ Memmapping (memory mapping) speeds up cache looking when reloading large numpy arrays:: >>> cachedir2 = 'your_cachedir2_location' >>> memory2 = Memory(cachedir2, mmap_mode='r') >>> square = memory2.cache(np.square) >>> a = np.vander(np.arange(3)).astype(float) >>> square(a) ________________________________________________________________________________ [Memory] Calling square... square(array([[0., 0., 1.], [1., 1., 1.], [4., 2., 1.]])) ___________________________________________________________square - ...min memmap([[ 0., 0., 1.], [ 1., 1., 1.], [16., 4., 1.]]) .. note:: Notice the debug mode used in the above example. It is useful for tracing of what is being reexecuted, and where the time is spent. If the `square` function is called with the same input argument, its return value is loaded from the disk using memmapping:: >>> res = square(a) >>> print(repr(res)) memmap([[ 0., 0., 1.], [ 1., 1., 1.], [16., 4., 1.]]) .. The memmap file must be closed to avoid file locking on Windows; closing numpy.memmap objects is done with del, which flushes changes to the disk >>> del res .. note:: If the memory mapping mode used was 'r', as in the above example, the array will be read only, and will be impossible to modified in place. On the other hand, using 'r+' or 'w+' will enable modification of the array, but will propagate these modification to the disk, which will corrupt the cache. If you want modification of the array in memory, we suggest you use the 'c' mode: copy on write. Shelving: using references to cached values ------------------------------------------- In some cases, it can be useful to get a reference to the cached result, instead of having the result itself. A typical example of this is when a lot of large numpy arrays must be dispatched across several workers: instead of sending the data themselves over the network, send a reference to the joblib cache, and let the workers read the data from a network filesystem, potentially taking advantage of some system-level caching too. Getting a reference to the cache can be done using the `call_and_shelve` method on the wrapped function:: >>> result = g.call_and_shelve(4) A long-running calculation, with parameter 4 >>> result #doctest: +ELLIPSIS MemorizedResult(location="...", func="...g...", args_id="...") Once computed, the output of `g` is stored on disk, and deleted from memory. Reading the associated value can then be performed with the `get` method:: >>> result.get() array([0.08, 0.77, 0.77, 0.08]) The cache for this particular value can be cleared using the `clear` method. Its invocation causes the stored value to be erased from disk. Any subsequent call to `get` will cause a `KeyError` exception to be raised:: >>> result.clear() >>> result.get() #doctest: +SKIP Traceback (most recent call last): ... KeyError: 'Non-existing cache value (may have been cleared).\nFile ... does not exist' A `MemorizedResult` instance contains all that is necessary to read the cached value. It can be pickled for transmission or storage, and the printed representation can even be copy-pasted to a different python interpreter. .. topic:: Shelving when cache is disabled In the case where caching is disabled (e.g. `Memory(None)`), the `call_and_shelve` method returns a `NotMemorizedResult` instance, that stores the full function output, instead of just a reference (since there is nothing to point to). All the above remains valid though, except for the copy-pasting feature. Gotchas ------- * **Across sessions, function cache is identified by the function's name**. Thus assigning the same name to different functions, their cache will override each-others (e.g. there are 'name collisions'), and unwanted re-run will happen:: >>> @memory.cache ... def func(x): ... print('Running func(%s)' % x) >>> func2 = func >>> @memory.cache ... def func(x): ... print('Running a different func(%s)' % x) As long as the same session is used, there are no collisions (in joblib 0.8 and above), although joblib does warn you that you are doing something dangerous:: >>> func(1) Running a different func(1) >>> # FIXME: The next line should create a JolibCollisionWarning but does not >>> # memory.rst:0: JobLibCollisionWarning: Possible name collisions between functions 'func' (:...) and 'func' (:...) >>> func2(1) #doctest: +ELLIPSIS Running func(1) >>> func(1) # No recomputation so far >>> func2(1) # No recomputation so far .. Empty the in-memory cache to simulate exiting and reloading the interpreter >>> import joblib.memory >>> joblib.memory._FUNCTION_HASHES.clear() But suppose the interpreter is exited and then restarted, the cache will not be identified properly, and the functions will be rerun:: >>> # FIXME: The next line will should create a JoblibCollisionWarning but does not. Also it is skipped because it does not produce any output >>> # memory.rst:0: JobLibCollisionWarning: Possible name collisions between functions 'func' (:...) and 'func' (:...) >>> func(1) #doctest: +ELLIPSIS +SKIP Running a different func(1) >>> func2(1) #doctest: +ELLIPSIS +SKIP Running func(1) As long as the same session is used, there are no needless recomputation:: >>> func(1) # No recomputation now >>> func2(1) # No recomputation now * **lambda functions** Beware that with Python 2.7 lambda functions cannot be separated out:: >>> def my_print(x): ... print(x) >>> f = memory.cache(lambda : my_print(1)) >>> g = memory.cache(lambda : my_print(2)) >>> f() 1 >>> f() >>> g() # doctest: +SKIP memory.rst:0: JobLibCollisionWarning: Cannot detect name collisions for function '' 2 >>> g() # doctest: +SKIP >>> f() # doctest: +SKIP 1 * **memory cannot be used on some complex objects**, e.g. a callable object with a `__call__` method. However, it works on numpy ufuncs:: >>> sin = memory.cache(np.sin) >>> print(sin(0)) 0.0 * **caching methods: memory is designed for pure functions and it is not recommended to use it for methods**. If one wants to use cache inside a class the recommended pattern is to cache a pure function and use the cached function inside your class, i.e. something like this:: @memory.cache def compute_func(arg1, arg2, arg3): # long computation return result class Foo(object): def __init__(self, args): self.data = None def compute(self): self.data = compute_func(self.arg1, self.arg2, 40) Using ``Memory`` for methods is not recommended and has some caveats that make it very fragile from a maintenance point of view because it is very easy to forget about these caveats when a software evolves. If this cannot be avoided (we would be interested about your use case by the way), here are a few known caveats: 1. a method cannot be decorated at class definition, because when the class is instantiated, the first argument (self) is *bound*, and no longer accessible to the `Memory` object. The following code won't work:: class Foo(object): @memory.cache # WRONG def method(self, args): pass The right way to do this is to decorate at instantiation time:: class Foo(object): def __init__(self, args): self.method = memory.cache(self.method) def method(self, ...): pass 2. The cached method will have ``self`` as one of its arguments. That means that the result will be recomputed if anything with ``self`` changes. For example if ``self.attr`` has changed calling ``self.method`` will recompute the result even if ``self.method`` does not use ``self.attr`` in its body. Another example is changing ``self`` inside the body of ``self.method``. The consequence is that ``self.method`` will create cache that will not be reused in subsequent calls. To alleviate these problems and if you *know* that the result of ``self.method`` does not depend on ``self`` you can use ``self.method = memory.cache(self.method, ignore=['self'])``. * **joblib cache entries may be invalidated after environment updates**. Values returned by :func:`joblib.hash` are not guaranteed to stay constant across ``joblib`` versions. This means that **all** entries of a :class:`Memory` cache can get invalidated when upgrading ``joblib``. Invalidation can also happen when upgrading a third party library (such as ``numpy``): in such a case, only the cached function calls with parameters that are constructs (or contain references to constructs) defined in the upgraded library should potentially be invalidated after the upgrade. Ignoring some arguments ----------------------- It may be useful not to recalculate a function when certain arguments change, for instance a debug flag. :class:`Memory` provides the ``ignore`` list:: >>> @memory.cache(ignore=['debug']) ... def my_func(x, debug=True): ... print('Called with x = %s' % x) >>> my_func(0) Called with x = 0 >>> my_func(0, debug=False) >>> my_func(0, debug=True) >>> # my_func was not reevaluated Custom cache validation ----------------------- In some cases, external factors can invalidate the cached results and one wants to have more control on whether to reuse a result or not. This is for instance the case if the results depends on database records that change over time: a small delay in the updates might be tolerable but after a while, the results might be invalid. One can have a finer control on the cache validity specifying a function via ``cache_validation_callback`` in :meth:`~joblib.Memory.cache`. For instance, one can only cache results that take more than 1s to be computed. >>> import time >>> def cache_validation_cb(metadata): ... # Only retrieve cached results for calls that take more than 1s ... return metadata['duration'] > 1 >>> @memory.cache(cache_validation_callback=cache_validation_cb) ... def my_func(delay=0): ... time.sleep(delay) ... print(f'Called with {delay}s delay') >>> my_func() Called with 0s delay >>> my_func(1.1) Called with 1.1s delay >>> my_func(1.1) # This result is retrieved from cache >>> my_func() # This one is not and the call is repeated Called with 0s delay ``cache_validation_cb`` will be called with a single argument containing the metadata of the cached call as a dictionary containing the following keys: - ``duration``: the duration of the function call, - ``time``: the timestamp when the cache called has been recorded - ``input_args``: a dictionary of keywords arguments for the cached function call. Note a validity duration for cached results can be defined via :func:`joblib.expires_after` by providing similar with arguments similar to the ones of a ``datetime.timedelta``: >>> from joblib import expires_after >>> @memory.cache(cache_validation_callback=expires_after(seconds=0.5)) ... def my_func(): ... print(f'Function run') >>> my_func() Function run >>> my_func() >>> time.sleep(0.5) >>> my_func() Function run .. _memory_reference: Reference documentation of the :class:`~joblib.Memory` class ------------------------------------------------------------ .. autoclass:: joblib.Memory :members: __init__, cache, eval, clear, reduce_size, format :no-inherited-members: :noindex: Useful methods of decorated functions ------------------------------------- Functions decorated by :meth:`Memory.cache ` are :class:`MemorizedFunc` objects that, in addition of behaving like normal functions, expose methods useful for cache exploration and management. For example, you can use :meth:`func.check_call_in_cache ` to check if a cache hit will occur for a decorated ``func`` given a set of inputs without actually needing to call the function itself:: >>> @memory.cache ... def func(x): ... print('Running func(%s)' % x) ... return x >>> type(func) >>> func(1) Running func(1) 1 >>> func.check_call_in_cache(1) # cache hit True >>> func.check_call_in_cache(2) # cache miss False .. autoclass:: MemorizedFunc :members: __init__, call, clear, check_call_in_cache .. Let us not forget to clean our cache dir once we are finished:: >>> import shutil >>> try: ... shutil.rmtree(cachedir) ... shutil.rmtree(cachedir2) ... except OSError: ... pass # this can sometimes fail under Windows Helper Reference ~~~~~~~~~~~~~~~~ .. autofunction:: joblib.expires_after joblib-1.3.2/doc/parallel.rst000066400000000000000000000452241446465525000160730ustar00rootroot00000000000000 .. _parallel: ================================= Embarrassingly parallel for loops ================================= Common usage ============ Joblib provides a simple helper class to write parallel for loops using multiprocessing. The core idea is to write the code to be executed as a generator expression, and convert it to parallel computing:: >>> from math import sqrt >>> [sqrt(i ** 2) for i in range(10)] [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] can be spread over 2 CPUs using the following:: >>> from math import sqrt >>> from joblib import Parallel, delayed >>> Parallel(n_jobs=2)(delayed(sqrt)(i ** 2) for i in range(10)) [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] The output can be a generator that yields the results as soon as they're available, even if the subsequent tasks aren't completed yet. The order of the outputs always matches the order the inputs have been submitted with:: >>> from math import sqrt >>> from joblib import Parallel, delayed >>> parallel = Parallel(n_jobs=2, return_as="generator") >>> output_generator = parallel(delayed(sqrt)(i ** 2) for i in range(10)) >>> print(type(output_generator)) >>> print(next(output_generator)) 0.0 >>> print(next(output_generator)) 1.0 >>> print(list(output_generator)) [2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] This generator enables reducing the memory footprint of :class:`joblib.Parallel` calls in case the results can benefit from on-the-fly aggregation, as illustrated in :ref:`sphx_glr_auto_examples_parallel_generator.py`. Future releases are planned to also support returning a generator that yields the results in the order of completion rather than the order of submission, by using ``return_as="unordered_generator"`` instead of ``return_as="generator"``. In this case the order of the outputs will depend on the concurrency of workers and will not be guaranteed to be deterministic, meaning the results can be yielded with a different order every time the code is executed. Thread-based parallelism vs process-based parallelism ===================================================== By default :class:`joblib.Parallel` uses the ``'loky'`` backend module to start separate Python worker processes to execute tasks concurrently on separate CPUs. This is a reasonable default for generic Python programs but can induce a significant overhead as the input and output data need to be serialized in a queue for communication with the worker processes (see :ref:`serialization_and_processes`). When you know that the function you are calling is based on a compiled extension that releases the Python Global Interpreter Lock (GIL) during most of its computation then it is more efficient to use threads instead of Python processes as concurrent workers. For instance this is the case if you write the CPU intensive part of your code inside a `with nogil`_ block of a Cython function. .. _`with nogil`: http://docs.cython.org/src/userguide/external_C_code.html#acquiring-and-releasing-the-gil To hint that your code can efficiently use threads, just pass ``prefer="threads"`` as parameter of the :class:`joblib.Parallel` constructor. In this case joblib will automatically use the ``"threading"`` backend instead of the default ``"loky"`` backend: >>> Parallel(n_jobs=2, prefer="threads")( ... delayed(sqrt)(i ** 2) for i in range(10)) [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] The :func:`~joblib.parallel_config` context manager helps selecting a specific backend implementation or setting the default number of jobs: >>> from joblib import parallel_config >>> with parallel_config(backend='threading', n_jobs=2): ... Parallel()(delayed(sqrt)(i ** 2) for i in range(10)) [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] The latter is especially useful when calling a library that uses :class:`joblib.Parallel` internally without exposing backend selection as part of its public API. Note that the ``prefer="threads"`` option was introduced in joblib 0.12. In prior versions, the same effect could be achieved by hardcoding a specific backend implementation such as ``backend="threading"`` in the call to :class:`joblib.Parallel` but this is now considered a bad pattern (when done in a library) as it does not make it possible to override that choice with the :func:`~joblib.parallel_config` context manager. .. topic:: The loky backend may not always be available Some rare systems do not support multiprocessing (for instance Pyodide). In this case the loky backend is not available and the default backend falls back to threading. In addition to the builtin joblib backends, there are several cluster-specific backends you can use: * `Dask `_ backend for Dask clusters (see :ref:`sphx_glr_auto_examples_parallel_distributed_backend_simple.py` for an example), * `Ray `_ backend for Ray clusters, * `Joblib Apache Spark Backend `_ to distribute joblib tasks on a Spark cluster. .. _serialization_and_processes: Serialization & Processes ========================= To share function definition across multiple python processes, it is necessary to rely on a serialization protocol. The standard protocol in python is :mod:`pickle` but its default implementation in the standard library has several limitations. For instance, it cannot serialize functions which are defined interactively or in the :code:`__main__` module. To avoid this limitation, the ``loky`` backend now relies on |cloudpickle| to serialize python objects. |cloudpickle| is an alternative implementation of the pickle protocol which allows the serialization of a greater number of objects, in particular interactively defined functions. So for most usages, the loky ``backend`` should work seamlessly. The main drawback of |cloudpickle| is that it can be slower than the :mod:`pickle` module in the standard library. In particular, it is critical for large python dictionaries or lists, where the serialization time can be up to 100 times slower. There is two ways to alter the serialization process for the ``joblib`` to temper this issue: - If you are on an UNIX system, you can switch back to the old ``multiprocessing`` backend. With this backend, interactively defined functions can be shared with the worker processes using the fast :mod:`pickle`. The main issue with this solution is that using ``fork`` to start the process breaks the standard POSIX and can have weird interaction with third party libraries such as ``numpy`` and ``openblas``. - If you wish to use the ``loky`` backend with a different serialization library, you can set the ``LOKY_PICKLER=mod_pickle`` environment variable to use the ``mod_pickle`` as the serialization library for ``loky``. The module ``mod_pickle`` passed as an argument should be importable as ``import mod_pickle`` and should contain a ``Pickler`` object, which will be used to serialize to objects. It can be set to ``LOKY_PICKLER=pickle`` to use the pickling module from stdlib. The main drawback with ``LOKY_PICKLER=pickle`` is that interactively defined functions will not be serializable anymore. To cope with this, you can use this solution together with the :func:`joblib.wrap_non_picklable_objects` wrapper, which can be used as a decorator to locally enable using |cloudpickle| for specific objects. This way, you can have fast pickling of all python objects and locally enable slow pickling for interactive functions. An example is given in loky_wrapper_. .. |cloudpickle| raw:: html cloudpickle .. _loky_wrapper: auto_examples/serialization_and_wrappers.html Shared-memory semantics ======================= The default backend of joblib will run each function call in isolated Python processes, therefore they cannot mutate a common Python object defined in the main program. However if the parallel function really needs to rely on the shared memory semantics of threads, it should be made explicit with ``require='sharedmem'``, for instance: >>> shared_set = set() >>> def collect(x): ... shared_set.add(x) ... >>> Parallel(n_jobs=2, require='sharedmem')( ... delayed(collect)(i) for i in range(5)) [None, None, None, None, None] >>> sorted(shared_set) [0, 1, 2, 3, 4] Keep in mind that relying a on the shared-memory semantics is probably suboptimal from a performance point of view as concurrent access to a shared Python object will suffer from lock contention. Reusing a pool of workers ========================= Some algorithms require to make several consecutive calls to a parallel function interleaved with processing of the intermediate results. Calling :class:`joblib.Parallel` several times in a loop is sub-optimal because it will create and destroy a pool of workers (threads or processes) several times which can cause a significant overhead. For this case it is more efficient to use the context manager API of the :class:`joblib.Parallel` class to re-use the same pool of workers for several calls to the :class:`joblib.Parallel` object:: >>> with Parallel(n_jobs=2) as parallel: ... accumulator = 0. ... n_iter = 0 ... while accumulator < 1000: ... results = parallel(delayed(sqrt)(accumulator + i ** 2) ... for i in range(5)) ... accumulator += sum(results) # synchronization barrier ... n_iter += 1 ... >>> (accumulator, n_iter) # doctest: +ELLIPSIS (1136.596..., 14) Note that the ``'loky'`` backend now used by default for process-based parallelism automatically tries to maintain and reuse a pool of workers by it-self even for calls without the context manager. .. include:: parallel_numpy.rst Avoiding over-subscription of CPU resources ============================================ The computation parallelism relies on the usage of multiple CPUs to perform the operation simultaneously. When using more processes than the number of CPU on a machine, the performance of each process is degraded as there is less computational power available for each process. Moreover, when many processes are running, the time taken by the OS scheduler to switch between them can further hinder the performance of the computation. It is generally better to avoid using significantly more processes or threads than the number of CPUs on a machine. Some third-party libraries -- *e.g.* the BLAS runtime used by ``numpy`` -- internally manage a thread-pool to perform their computations. The default behavior is generally to use a number of threads equals to the number of CPUs available. When these libraries are used with :class:`joblib.Parallel`, each worker will spawn its own thread-pools, resulting in a massive over-subscription of resources that can slow down the computation compared to a sequential one. To cope with this problem, joblib tells supported third-party libraries to use a limited number of threads in workers managed by the ``'loky'`` backend: by default each worker process will have environment variables set to allow a maximum of ``cpu_count() // n_jobs`` so that the total number of threads used by all the workers does not exceed the number of CPUs of the host. This behavior can be overridden by setting the proper environment variables to the desired number of threads. This override is supported for the following libraries: - OpenMP with the environment variable ``'OMP_NUM_THREADS'``, - OpenBLAS with the ``'OPENBLAS_NUM_THREADS'``, - MKL with the environment variable ``'MKL_NUM_THREADS'``, - Accelerated with the environment variable ``'VECLIB_MAXIMUM_THREADS'``, - Numexpr with the environment variable ``'NUMEXPR_NUM_THREADS'``. Since joblib 0.14, it is also possible to programmatically override the default number of threads using the ``inner_max_num_threads`` argument of the :func:`~joblib.parallel_config` function as follows: .. code-block:: python from joblib import Parallel, delayed, parallel_config with parallel_config(backend="loky", inner_max_num_threads=2): results = Parallel(n_jobs=4)(delayed(func)(x, y) for x, y in data) In this example, 4 Python worker processes will be allowed to use 2 threads each, meaning that this program will be able to use up to 8 CPUs concurrently. Custom backend API ================== .. versionadded:: 0.10 User can provide their own implementation of a parallel processing backend in addition to the ``'loky'``, ``'threading'``, ``'multiprocessing'`` backends provided by default. A backend is registered with the :func:`joblib.register_parallel_backend` function by passing a name and a backend factory. The backend factory can be any callable that returns an instance of ``ParallelBackendBase``. Please refer to the `default backends source code`_ as a reference if you want to implement your own custom backend. .. _`default backends source code`: https://github.com/joblib/joblib/blob/master/joblib/_parallel_backends.py Note that it is possible to register a backend class that has some mandatory constructor parameters such as the network address and connection credentials for a remote cluster computing service:: class MyCustomBackend(ParallelBackendBase): def __init__(self, endpoint, api_key): self.endpoint = endpoint self.api_key = api_key ... # Do something with self.endpoint and self.api_key somewhere in # one of the method of the class register_parallel_backend('custom', MyCustomBackend) The connection parameters can then be passed to the :func:`~joblib.parallel_config` context manager:: with parallel_config(backend='custom', endpoint='http://compute', api_key='42'): Parallel()(delayed(some_function)(i) for i in range(10)) Using the context manager can be helpful when using a third-party library that uses :class:`joblib.Parallel` internally while not exposing the ``backend`` argument in its own API. A problem exists that external packages that register new parallel backends must now be imported explicitly for their backends to be identified by joblib:: >>> import joblib >>> with joblib.parallel_config(backend='custom'): # doctest: +SKIP ... ... # this fails KeyError: 'custom' # Import library to register external backend >>> import my_custom_backend_library # doctest: +SKIP >>> with joblib.parallel_config(backend='custom'): # doctest: +SKIP ... ... # this works This can be confusing for users. To resolve this, external packages can safely register their backends directly within the joblib codebase by creating a small function that registers their backend, and including this function within the ``joblib.parallel.EXTERNAL_PACKAGES`` dictionary:: def _register_custom(): try: import my_custom_library except ImportError: raise ImportError("an informative error message") EXTERNAL_BACKENDS['custom'] = _register_custom This is subject to community review, but can reduce the confusion for users when relying on side effects of external package imports. Old multiprocessing backend =========================== Prior to version 0.12, joblib used the ``'multiprocessing'`` backend as default backend instead of ``'loky'``. This backend creates an instance of `multiprocessing.Pool` that forks the Python interpreter in multiple processes to execute each of the items of the list. The `delayed` function is a simple trick to be able to create a tuple `(function, args, kwargs)` with a function-call syntax. .. warning:: Under Windows, the use of ``multiprocessing.Pool`` requires to protect the main loop of code to avoid recursive spawning of subprocesses when using :class:`joblib.Parallel`. In other words, you should be writing code like this when using the ``'multiprocessing'`` backend: .. code-block:: python import .... def function1(...): ... def function2(...): ... ... if __name__ == '__main__': # do stuff with imports and functions defined about ... **No** code should *run* outside of the ``"if __name__ == '__main__'"`` blocks, only imports and definitions. The ``'loky'`` backend used by default in joblib 0.12 and later does not impose this anymore. Bad interaction of multiprocessing and third-party libraries ============================================================ Using the ``'multiprocessing'`` backend can cause a crash when using third party libraries that manage their own native thread-pool if the library is first used in the main process and subsequently called again in a worker process (inside the :class:`joblib.Parallel` call). Joblib version 0.12 and later are no longer subject to this problem thanks to the use of `loky `_ as the new default backend for process-based parallelism. Prior to Python 3.4 the ``'multiprocessing'`` backend of joblib can only use the ``fork`` strategy to create worker processes under non-Windows systems. This can cause some third-party libraries to crash or freeze. Such libraries include Apple vecLib / Accelerate (used by NumPy under OSX), some old version of OpenBLAS (prior to 0.2.10) or the OpenMP runtime implementation from GCC which is used internally by third-party libraries such as XGBoost, spaCy, OpenCV... The best way to avoid this problem is to use the ``'loky'`` backend instead of the ``multiprocessing`` backend. Prior to joblib 0.12, it is also possible to get :class:`joblib.Parallel` configured to use the ``'forkserver'`` start method on Python 3.4 and later. The start method has to be configured by setting the ``JOBLIB_START_METHOD`` environment variable to ``'forkserver'`` instead of the default ``'fork'`` start method. However the user should be aware that using the ``'forkserver'`` method prevents :class:`joblib.Parallel` to call function interactively defined in a shell session. You can read more on this topic in the `multiprocessing documentation `_. Under Windows the ``fork`` system call does not exist at all so this problem does not exist (but multiprocessing has more overhead). `Parallel` reference documentation ================================== .. autoclass:: joblib.Parallel :members: dispatch_next, dispatch_one_batch, format, print_progress :no-inherited-members: :noindex: .. autofunction:: joblib.delayed .. autofunction:: joblib.parallel_config :noindex: .. autofunction:: joblib.wrap_non_picklable_objects .. autofunction:: joblib.register_parallel_backend .. autoclass:: joblib.parallel.ParallelBackendBase .. autoclass:: joblib.parallel.AutoBatchingMixinjoblib-1.3.2/doc/parallel_numpy.rst000066400000000000000000000142701446465525000173200ustar00rootroot00000000000000.. For doctests: >>> import sys >>> setup = getfixture('parallel_numpy_fixture') >>> fixture = setup(sys.modules[__name__]) Working with numerical data in shared memory (memmapping) ========================================================= By default the workers of the pool are real Python processes forked using the ``multiprocessing`` module of the Python standard library when ``n_jobs != 1``. The arguments passed as input to the ``Parallel`` call are serialized and reallocated in the memory of each worker process. This can be problematic for large arguments as they will be reallocated ``n_jobs`` times by the workers. As this problem can often occur in scientific computing with ``numpy`` based datastructures, :class:`joblib.Parallel` provides a special handling for large arrays to automatically dump them on the filesystem and pass a reference to the worker to open them as memory map on that file using the ``numpy.memmap`` subclass of ``numpy.ndarray``. This makes it possible to share a segment of data between all the worker processes. .. note:: The following only applies with the ``"loky"` and ``'multiprocessing'`` process-backends. If your code can release the GIL, then using a thread-based backend by passing ``prefer='threads'`` is even more efficient because it makes it possible to avoid the communication overhead of process-based parallelism. Scientific Python libraries such as numpy, scipy, pandas and scikit-learn often release the GIL in performance critical code paths. It is therefore advised to always measure the speed of thread-based parallelism and use it when the scalability is not limited by the GIL. Automated array to memmap conversion ------------------------------------ The automated array to memmap conversion is triggered by a configurable threshold on the size of the array:: >>> import numpy as np >>> from joblib import Parallel, delayed >>> def is_memmap(obj): ... return isinstance(obj, np.memmap) >>> Parallel(n_jobs=2, max_nbytes=1e6)( ... delayed(is_memmap)(np.ones(int(i))) ... for i in [1e2, 1e4, 1e6]) [False, False, True] By default the data is dumped to the ``/dev/shm`` shared-memory partition if it exists and is writable (typically the case under Linux). Otherwise the operating system's temporary folder is used. The location of the temporary data files can be customized by passing a ``temp_folder`` argument to the ``Parallel`` constructor. Passing ``max_nbytes=None`` makes it possible to disable the automated array to memmap conversion. Manual management of memmapped input data ----------------------------------------- For even finer tuning of the memory usage it is also possible to dump the array as a memmap directly from the parent process to free the memory before forking the worker processes. For instance let's allocate a large array in the memory of the parent process:: >>> large_array = np.ones(int(1e6)) Dump it to a local file for memmapping:: >>> import tempfile >>> import os >>> from joblib import load, dump >>> temp_folder = tempfile.mkdtemp() >>> filename = os.path.join(temp_folder, 'joblib_test.mmap') >>> if os.path.exists(filename): os.unlink(filename) >>> _ = dump(large_array, filename) >>> large_memmap = load(filename, mmap_mode='r+') The ``large_memmap`` variable is pointing to a ``numpy.memmap`` instance:: >>> large_memmap.__class__.__name__, large_array.nbytes, large_array.shape ('memmap', 8000000, (1000000,)) >>> np.allclose(large_array, large_memmap) True The original array can be freed from the main process memory:: >>> del large_array >>> import gc >>> _ = gc.collect() It is possible to slice ``large_memmap`` into a smaller memmap:: >>> small_memmap = large_memmap[2:5] >>> small_memmap.__class__.__name__, small_memmap.nbytes, small_memmap.shape ('memmap', 24, (3,)) Finally a ``np.ndarray`` view backed on that same memory mapped file can be used:: >>> small_array = np.asarray(small_memmap) >>> small_array.__class__.__name__, small_array.nbytes, small_array.shape ('ndarray', 24, (3,)) All those three datastructures point to the same memory buffer and this same buffer will also be reused directly by the worker processes of a ``Parallel`` call:: >>> Parallel(n_jobs=2, max_nbytes=None)( ... delayed(is_memmap)(a) ... for a in [large_memmap, small_memmap, small_array]) [True, True, True] Note that here ``max_nbytes=None`` is used to disable the auto-dumping feature of ``Parallel``. ``small_array`` is still in shared memory in the worker processes because it was already backed by shared memory in the parent process. The pickling machinery of ``Parallel`` multiprocessing queues are able to detect this situation and optimize it on the fly to limit the number of memory copies. Writing parallel computation results in shared memory ----------------------------------------------------- If data are opened using the ``w+`` or ``r+`` mode in the main program, the worker will get ``r+`` mode access. Thus the worker will be able to write its results directly to the original data, alleviating the need of the serialization to send back the results to the parent process. Here is an example script on parallel processing with preallocated ``numpy.memmap`` datastructures :ref:`sphx_glr_auto_examples_parallel_memmap.py`. .. warning:: Having concurrent workers write on overlapping shared memory data segments, for instance by using inplace operators and assignments on a `numpy.memmap` instance, can lead to data corruption as numpy does not offer atomic operations. The previous example does not risk that issue as each task is updating an exclusive segment of the shared result array. Some C/C++ compilers offer lock-free atomic primitives such as add-and-fetch or compare-and-swap that could be exposed to Python via CFFI_ for instance. However providing numpy-aware atomic constructs is outside of the scope of the joblib project. .. _CFFI: https://cffi.readthedocs.org A final note: don't forget to clean up any temporary folder when you are done with the computation:: >>> import shutil >>> try: ... shutil.rmtree(temp_folder) ... except OSError: ... pass # this can sometimes fail under Windows joblib-1.3.2/doc/persistence.rst000066400000000000000000000161171446465525000166220ustar00rootroot00000000000000.. _persistence: =========== Persistence =========== .. currentmodule:: joblib.numpy_pickle Use case ======== :func:`joblib.dump` and :func:`joblib.load` provide a replacement for pickle to work efficiently on arbitrary Python objects containing large data, in particular large numpy arrays. .. warning:: :func:`joblib.dump` and :func:`joblib.load` are based on the Python pickle serialization model, which means that arbitrary Python code can be executed when loading a serialized object with :func:`joblib.load`. :func:`joblib.load` should therefore never be used to load objects from an untrusted source or otherwise you will introduce a security vulnerability in your program. .. note:: As of Python 3.8 and numpy 1.16, pickle protocol 5 introduced in `PEP 574 `_ supports efficient serialization and de-serialization for large data buffers natively using the standard library:: pickle.dump(large_object, fileobj, protocol=5) A simple example ================ First create a temporary directory:: >>> from tempfile import mkdtemp >>> savedir = mkdtemp() >>> import os >>> filename = os.path.join(savedir, 'test.joblib') Then create an object to be persisted:: >>> import numpy as np >>> to_persist = [('a', [1, 2, 3]), ('b', np.arange(10))] which is saved into `filename`:: >>> import joblib >>> joblib.dump(to_persist, filename) # doctest: +ELLIPSIS ['...test.joblib'] The object can then be reloaded from the file:: >>> joblib.load(filename) [('a', [1, 2, 3]), ('b', array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))] Persistence in file objects =========================== Instead of filenames, :func:`joblib.dump` and :func:`joblib.load` functions also accept file objects: >>> with open(filename, 'wb') as fo: # doctest: +ELLIPSIS ... joblib.dump(to_persist, fo) >>> with open(filename, 'rb') as fo: # doctest: +ELLIPSIS ... joblib.load(fo) [('a', [1, 2, 3]), ('b', array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))] Compressed joblib pickles ========================= Setting the `compress` argument to `True` in :func:`joblib.dump` will allow to save space on disk: >>> joblib.dump(to_persist, filename + '.compressed', compress=True) # doctest: +ELLIPSIS ['...test.joblib.compressed'] If the filename extension corresponds to one of the supported compression methods, the compressor will be used automatically: >>> joblib.dump(to_persist, filename + '.z') # doctest: +ELLIPSIS ['...test.joblib.z'] By default, :func:`joblib.dump` uses the zlib compression method as it gives the best tradeoff between speed and disk space. The other supported compression methods are 'gzip', 'bz2', 'lzma' and 'xz': >>> # Dumping in a gzip compressed file using a compress level of 3. >>> joblib.dump(to_persist, filename + '.gz', compress=('gzip', 3)) # doctest: +ELLIPSIS ['...test.joblib.gz'] >>> joblib.load(filename + '.gz') [('a', [1, 2, 3]), ('b', array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))] >>> joblib.dump(to_persist, filename + '.bz2', compress=('bz2', 3)) # doctest: +ELLIPSIS ['...test.joblib.bz2'] >>> joblib.load(filename + '.bz2') [('a', [1, 2, 3]), ('b', array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))] The ``compress`` parameter of the :func:`joblib.dump` function also accepts a string corresponding to the name of the compressor used. When using this, the default compression level is used by the compressor: >>> joblib.dump(to_persist, filename + '.gz', compress='gzip') # doctest: +ELLIPSIS ['...test.joblib.gz'] .. note:: Lzma and Xz compression methods are only available for python versions >= 3.3. Compressor files provided by the python standard library can also be used to compress pickle, e.g ``gzip.GzipFile``, ``bz2.BZ2File``, ``lzma.LZMAFile``: >>> # Dumping in a gzip.GzipFile object using a compression level of 3. >>> import gzip >>> with gzip.GzipFile(filename + '.gz', 'wb', compresslevel=3) as fo: # doctest: +ELLIPSIS ... joblib.dump(to_persist, fo) >>> with gzip.GzipFile(filename + '.gz', 'rb') as fo: # doctest: +ELLIPSIS ... joblib.load(fo) [('a', [1, 2, 3]), ('b', array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))] If the ``lz4`` package is installed, this compression method is automatically available with the dump function. >>> joblib.dump(to_persist, filename + '.lz4') # doctest: +ELLIPSIS ['...test.joblib.lz4'] >>> joblib.load(filename + '.lz4') [('a', [1, 2, 3]), ('b', array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))] .. note:: LZ4 compression is only available with python major versions >= 3 More details can be found in the :func:`joblib.dump` and :func:`joblib.load` documentation. Registering extra compressors ----------------------------- Joblib provides :func:`joblib.register_compressor` in order to extend the list of default compressors available. To fit with Joblib internal implementation and features, such as :func:`joblib.load` and :class:`joblib.Memory`, the registered compressor should implement the Python file object interface. Compatibility across python versions ------------------------------------ Compatibility of joblib pickles across python versions is not fully supported. Note that, for a very restricted set of objects, this may appear to work when saving a pickle with python 2 and loading it with python 3 but relying on it is strongly discouraged. If you are switching between python versions, you will need to save a different joblib pickle for each python version. Here are a few examples or exceptions: - Saving joblib pickle with python 2, trying to load it with python 3:: Traceback (most recent call last): File "/home/lesteve/dev/joblib/joblib/numpy_pickle.py", line 453, in load obj = unpickler.load() File "/home/lesteve/miniconda3/lib/python3.4/pickle.py", line 1038, in load dispatch[key[0]](self) File "/home/lesteve/miniconda3/lib/python3.4/pickle.py", line 1176, in load_binstring self.append(self._decode_string(data)) File "/home/lesteve/miniconda3/lib/python3.4/pickle.py", line 1158, in _decode_string return value.decode(self.encoding, self.errors) UnicodeDecodeError: 'ascii' codec can't decode byte 0x80 in position 1024: ordinal not in range(128) Traceback (most recent call last): File "", line 1, in File "/home/lesteve/dev/joblib/joblib/numpy_pickle.py", line 462, in load raise new_exc ValueError: You may be trying to read with python 3 a joblib pickle generated with python 2. This is not feature supported by joblib. - Saving joblib pickle with python 3, trying to load it with python 2:: Traceback (most recent call last): File "", line 1, in File "joblib/numpy_pickle.py", line 453, in load obj = unpickler.load() File "/home/lesteve/miniconda3/envs/py27/lib/python2.7/pickle.py", line 858, in load dispatch[key](self) File "/home/lesteve/miniconda3/envs/py27/lib/python2.7/pickle.py", line 886, in load_proto raise ValueError, "unsupported pickle protocol: %d" % proto ValueError: unsupported pickle protocol: 3 joblib-1.3.2/doc/why.rst000066400000000000000000000032201446465525000150740ustar00rootroot00000000000000 Why joblib: project goals ========================= Benefits of pipelines --------------------- Pipeline processing systems can provide a set of useful features: Data-flow programming for performance ..................................... * **On-demand computing:** in pipeline systems such as labView or VTK, calculations are performed as needed by the outputs and only when inputs change. * **Transparent parallelization:** a pipeline topology can be inspected to deduce which operations can be run in parallel (it is equivalent to purely functional programming). Provenance tracking to understand the code .......................................... * **Tracking of data and computations:** This enables the reproducibility of a computational experiment. * **Inspecting data flow:** Inspecting intermediate results helps debugging and understanding. .. topic:: But pipeline frameworks can get in the way :class: warning Joblib's philosophy is to keep the underlying algorithm code unchanged, avoiding framework-style modifications. Joblib's approach ----------------- Functions are the simplest abstraction used by everyone. Pipeline jobs (or tasks) in Joblib are made of decorated functions. Tracking of parameters in a meaningful way requires specification of data model. Joblib gives up on that and uses hashing for performance and robustness. Design choices -------------- * No dependencies other than Python * Robust, well-tested code, at the cost of functionality * Fast and suitable for scientific computing on big dataset without changing the original code * Only local imports: **embed joblib in your code by copying it** joblib-1.3.2/examples/000077500000000000000000000000001446465525000146075ustar00rootroot00000000000000joblib-1.3.2/examples/README.txt000066400000000000000000000002031446465525000163000ustar00rootroot00000000000000.. _general_examples: Examples ======== General examples ---------------- General-purpose and introductory examples for joblib. joblib-1.3.2/examples/compressors_comparison.py000066400000000000000000000215241446465525000217760ustar00rootroot00000000000000""" =============================== Improving I/O using compressors =============================== This example compares the compressors available in Joblib. In the example, Zlib, LZMA and LZ4 compression only are used but Joblib also supports BZ2 and GZip compression methods. For each compared compression method, this example dumps and reloads a dataset fetched from an online machine-learning database. This gives 3 information: the size on disk of the compressed data, the time spent to dump and the time spent to reload the data from disk. """ import os import os.path import time ############################################################################### # Get some data from real-world use cases # --------------------------------------- # # First fetch the benchmark dataset from an online machine-learning database # and load it in a pandas dataframe. import pandas as pd url = "https://github.com/joblib/dataset/raw/main/kddcup.data.gz" names = ("duration, protocol_type, service, flag, src_bytes, " "dst_bytes, land, wrong_fragment, urgent, hot, " "num_failed_logins, logged_in, num_compromised, " "root_shell, su_attempted, num_root, " "num_file_creations, ").split(', ') data = pd.read_csv(url, names=names, nrows=1e6) ############################################################################### # Dump and load the dataset without compression # --------------------------------------------- # # This gives reference values for later comparison. from joblib import dump, load pickle_file = './pickle_data.joblib' ############################################################################### # Start by measuring the time spent for dumping the raw data: start = time.time() with open(pickle_file, 'wb') as f: dump(data, f) raw_dump_duration = time.time() - start print("Raw dump duration: %0.3fs" % raw_dump_duration) ############################################################################### # Then measure the size of the raw dumped data on disk: raw_file_size = os.stat(pickle_file).st_size / 1e6 print("Raw dump file size: %0.3fMB" % raw_file_size) ############################################################################### # Finally measure the time spent for loading the raw data: start = time.time() with open(pickle_file, 'rb') as f: load(f) raw_load_duration = time.time() - start print("Raw load duration: %0.3fs" % raw_load_duration) ############################################################################### # Dump and load the dataset using the Zlib compression method # ----------------------------------------------------------- # # The compression level is using the default value, 3, which is, in general, a # good compromise between compression and speed. ############################################################################### # Start by measuring the time spent for dumping of the zlib data: start = time.time() with open(pickle_file, 'wb') as f: dump(data, f, compress='zlib') zlib_dump_duration = time.time() - start print("Zlib dump duration: %0.3fs" % zlib_dump_duration) ############################################################################### # Then measure the size of the zlib dump data on disk: zlib_file_size = os.stat(pickle_file).st_size / 1e6 print("Zlib file size: %0.3fMB" % zlib_file_size) ############################################################################### # Finally measure the time spent for loading the compressed dataset: start = time.time() with open(pickle_file, 'rb') as f: load(f) zlib_load_duration = time.time() - start print("Zlib load duration: %0.3fs" % zlib_load_duration) ############################################################################### # .. note:: The compression format is detected automatically by Joblib. # The compression format is identified by the standard magic number present # at the beginning of the file. Joblib uses this information to determine # the compression method used. # This is the case for all compression methods supported by Joblib. ############################################################################### # Dump and load the dataset using the LZMA compression method # ----------------------------------------------------------- # # LZMA compression method has a very good compression rate but at the cost # of being very slow. # In this example, a light compression level, e.g. 3, is used to speed up a # bit the dump/load cycle. ############################################################################### # Start by measuring the time spent for dumping the lzma data: start = time.time() with open(pickle_file, 'wb') as f: dump(data, f, compress=('lzma', 3)) lzma_dump_duration = time.time() - start print("LZMA dump duration: %0.3fs" % lzma_dump_duration) ############################################################################### # Then measure the size of the lzma dump data on disk: lzma_file_size = os.stat(pickle_file).st_size / 1e6 print("LZMA file size: %0.3fMB" % lzma_file_size) ############################################################################### # Finally measure the time spent for loading the lzma data: start = time.time() with open(pickle_file, 'rb') as f: load(f) lzma_load_duration = time.time() - start print("LZMA load duration: %0.3fs" % lzma_load_duration) ############################################################################### # Dump and load the dataset using the LZ4 compression method # ---------------------------------------------------------- # # LZ4 compression method is known to be one of the fastest available # compression method but with a compression rate a bit lower than Zlib. In # most of the cases, this method is a good choice. ############################################################################### # .. note:: In order to use LZ4 compression with Joblib, the # `lz4 `_ package must be installed # on the system. ############################################################################### # Start by measuring the time spent for dumping the lz4 data: start = time.time() with open(pickle_file, 'wb') as f: dump(data, f, compress='lz4') lz4_dump_duration = time.time() - start print("LZ4 dump duration: %0.3fs" % lz4_dump_duration) ############################################################################### # Then measure the size of the lz4 dump data on disk: lz4_file_size = os.stat(pickle_file).st_size / 1e6 print("LZ4 file size: %0.3fMB" % lz4_file_size) ############################################################################### # Finally measure the time spent for loading the lz4 data: start = time.time() with open(pickle_file, 'rb') as f: load(f) lz4_load_duration = time.time() - start print("LZ4 load duration: %0.3fs" % lz4_load_duration) ############################################################################### # Comparing the results # --------------------- import numpy as np import matplotlib.pyplot as plt N = 4 load_durations = (raw_load_duration, lz4_load_duration, zlib_load_duration, lzma_load_duration) dump_durations = (raw_dump_duration, lz4_dump_duration, zlib_dump_duration, lzma_dump_duration) file_sizes = (raw_file_size, lz4_file_size, zlib_file_size, lzma_file_size) ind = np.arange(N) width = 0.5 plt.figure(1, figsize=(5, 4)) p1 = plt.bar(ind, dump_durations, width) p2 = plt.bar(ind, load_durations, width, bottom=dump_durations) plt.ylabel('Time in seconds') plt.title('Dump and load durations') plt.xticks(ind, ('Raw', 'LZ4', 'Zlib', 'LZMA')) plt.yticks(np.arange(0, lzma_load_duration + lzma_dump_duration)) plt.legend((p1[0], p2[0]), ('Dump duration', 'Load duration')) ############################################################################### # Compared with other compressors, LZ4 is clearly the fastest, especially for # dumping compressed data on disk. In this particular case, it can even be # faster than the raw dump. # Also note that dump and load durations depend on the I/O speed of the # underlying storage: for example, with SSD hard drives the LZ4 compression # will be slightly slower than raw dump/load, whereas with spinning hard disk # drives (HDD) or remote storage (NFS), LZ4 is faster in general. # # LZMA and Zlib, even if always slower for dumping data, are quite fast when # re-loading compressed data from disk. plt.figure(2, figsize=(5, 4)) plt.bar(ind, file_sizes, width, log=True) plt.ylabel('File size in MB') plt.xticks(ind, ('Raw', 'LZ4', 'Zlib', 'LZMA')) ############################################################################### # Compressed data obviously takes a lot less space on disk than raw data. LZMA # is the best compression method in terms of compression rate. Zlib also has a # better compression rate than LZ4. plt.show() ############################################################################### # Clear the pickle file # --------------------- import os os.remove(pickle_file) joblib-1.3.2/examples/memory_basic_usage.py000066400000000000000000000111271446465525000210200ustar00rootroot00000000000000""" ======================== How to use joblib.Memory ======================== This example illustrates the usage of :class:`joblib.Memory` with both functions and methods. """ ############################################################################### # Without :class:`joblib.Memory` ############################################################################### # # ``costly_compute`` emulates a computationally expensive process which later # will benefit from caching using :class:`joblib.Memory`. import time import numpy as np def costly_compute(data, column_index=0): """Simulate an expensive computation""" time.sleep(5) return data[column_index] ############################################################################### # Be sure to set the random seed to generate deterministic data. Indeed, if the # data is not deterministic, the :class:`joblib.Memory` instance will not be # able to reuse the cache from one run to another. rng = np.random.RandomState(42) data = rng.randn(int(1e5), 10) start = time.time() data_trans = costly_compute(data) end = time.time() print('\nThe function took {:.2f} s to compute.'.format(end - start)) print('\nThe transformed data are:\n {}'.format(data_trans)) ############################################################################### # Caching the result of a function to avoid recomputing ############################################################################### # # If we need to call our function several time with the same input data, it is # beneficial to avoid recomputing the same results over and over since it is # expensive. :class:`joblib.Memory` enables to cache results from a function # into a specific location. from joblib import Memory location = './cachedir' memory = Memory(location, verbose=0) def costly_compute_cached(data, column_index=0): """Simulate an expensive computation""" time.sleep(5) return data[column_index] costly_compute_cached = memory.cache(costly_compute_cached) start = time.time() data_trans = costly_compute_cached(data) end = time.time() print('\nThe function took {:.2f} s to compute.'.format(end - start)) print('\nThe transformed data are:\n {}'.format(data_trans)) ############################################################################### # At the first call, the results will be cached. Therefore, the computation # time corresponds to the time to compute the results plus the time to dump the # results into the disk. start = time.time() data_trans = costly_compute_cached(data) end = time.time() print('\nThe function took {:.2f} s to compute.'.format(end - start)) print('\nThe transformed data are:\n {}'.format(data_trans)) ############################################################################### # At the second call, the computation time is largely reduced since the results # are obtained by loading the data previously dumped to the disk instead of # recomputing the results. ############################################################################### # Using :class:`joblib.Memory` with a method ############################################################################### # # :class:`joblib.Memory` is designed to work with functions with no side # effects. When dealing with class, the computationally expensive part of a # method has to be moved to a function and decorated in the class method. def _costly_compute_cached(data, column): time.sleep(5) return data[column] class Algorithm(object): """A class which is using the previous function.""" def __init__(self, column=0): self.column = column def transform(self, data): costly_compute = memory.cache(_costly_compute_cached) return costly_compute(data, self.column) transformer = Algorithm() start = time.time() data_trans = transformer.transform(data) end = time.time() print('\nThe function took {:.2f} s to compute.'.format(end - start)) print('\nThe transformed data are:\n {}'.format(data_trans)) ############################################################################### start = time.time() data_trans = transformer.transform(data) end = time.time() print('\nThe function took {:.2f} s to compute.'.format(end - start)) print('\nThe transformed data are:\n {}'.format(data_trans)) ############################################################################### # As expected, the second call to the ``transform`` method load the results # which have been cached. ############################################################################### # Clean up cache directory ############################################################################### memory.clear(warn=False) joblib-1.3.2/examples/nested_parallel_memory.py000066400000000000000000000117441446465525000217160ustar00rootroot00000000000000""" ================================================== Checkpoint using joblib.Memory and joblib.Parallel ================================================== This example illustrates how to cache intermediate computing results using :class:`joblib.Memory` within :class:`joblib.Parallel`. """ ############################################################################### # Embed caching within parallel processing ############################################################################### # # It is possible to cache a computationally expensive function executed during # a parallel process. ``costly_compute`` emulates such time consuming function. import time def costly_compute(data, column): """Emulate a costly function by sleeping and returning a column.""" time.sleep(2) return data[column] def data_processing_mean(data, column): """Compute the mean of a column.""" return costly_compute(data, column).mean() ############################################################################### # Create some data. The random seed is fixed to generate deterministic data # across Python session. Note that this is not necessary for this specific # example since the memory cache is cleared at the end of the session. import numpy as np rng = np.random.RandomState(42) data = rng.randn(int(1e4), 4) ############################################################################### # It is first possible to make the processing without caching or parallel # processing. start = time.time() results = [data_processing_mean(data, col) for col in range(data.shape[1])] stop = time.time() print('\nSequential processing') print('Elapsed time for the entire processing: {:.2f} s' .format(stop - start)) ############################################################################### # ``costly_compute`` is expensive to compute and it is used as an intermediate # step in ``data_processing_mean``. Therefore, it is interesting to store the # intermediate results from ``costly_compute`` using :class:`joblib.Memory`. from joblib import Memory location = './cachedir' memory = Memory(location, verbose=0) costly_compute_cached = memory.cache(costly_compute) ############################################################################### # Now, we define ``data_processing_mean_using_cache`` which benefits from the # cache by calling ``costly_compute_cached`` def data_processing_mean_using_cache(data, column): """Compute the mean of a column.""" return costly_compute_cached(data, column).mean() ############################################################################### # Then, we execute the same processing in parallel and caching the intermediate # results. from joblib import Parallel, delayed start = time.time() results = Parallel(n_jobs=2)( delayed(data_processing_mean_using_cache)(data, col) for col in range(data.shape[1])) stop = time.time() print('\nFirst round - caching the data') print('Elapsed time for the entire processing: {:.2f} s' .format(stop - start)) ############################################################################### # By using 2 workers, the parallel processing gives a x2 speed-up compared to # the sequential case. By executing again the same process, the intermediate # results obtained by calling ``costly_compute_cached`` will be loaded from the # cache instead of executing the function. start = time.time() results = Parallel(n_jobs=2)( delayed(data_processing_mean_using_cache)(data, col) for col in range(data.shape[1])) stop = time.time() print('\nSecond round - reloading from the cache') print('Elapsed time for the entire processing: {:.2f} s' .format(stop - start)) ############################################################################### # Reuse intermediate checkpoints ############################################################################### # # Having cached the intermediate results of the ``costly_compute_cached`` # function, they are reusable by calling the function. We define a new # processing which will take the maximum of the array returned by # ``costly_compute_cached`` instead of previously the mean. def data_processing_max_using_cache(data, column): """Compute the max of a column.""" return costly_compute_cached(data, column).max() start = time.time() results = Parallel(n_jobs=2)( delayed(data_processing_max_using_cache)(data, col) for col in range(data.shape[1])) stop = time.time() print('\nReusing intermediate checkpoints') print('Elapsed time for the entire processing: {:.2f} s' .format(stop - start)) ############################################################################### # The processing time only corresponds to the execution of the ``max`` # function. The internal call to ``costly_compute_cached`` is reloading the # results from the cache. ############################################################################### # Clean-up the cache folder ############################################################################### memory.clear(warn=False) joblib-1.3.2/examples/parallel/000077500000000000000000000000001446465525000164035ustar00rootroot00000000000000joblib-1.3.2/examples/parallel/README.txt000066400000000000000000000001571446465525000201040ustar00rootroot00000000000000.. _parallel_examples: Parallel examples ----------------- Examples demoing more advanced parallel patterns. joblib-1.3.2/examples/parallel/distributed_backend_simple.py000066400000000000000000000037001446465525000243170ustar00rootroot00000000000000""" Using Dask for single-machine parallel computing ================================================ This example shows the simplest usage of the `Dask `_ backend on your local machine. This is useful for prototyping a solution, to later be run on a truly `distributed Dask cluster `_, as the only change needed is the cluster class. Another realistic usage scenario: combining dask code with joblib code, for instance using dask for preprocessing data, and scikit-learn for machine learning. In such a setting, it may be interesting to use distributed as a backend scheduler for both dask and joblib, to orchestrate the computation. """ ############################################################################### # Setup the distributed client ############################################################################### from dask.distributed import Client, LocalCluster # replace with whichever cluster class you're using # https://docs.dask.org/en/stable/deploying.html#distributed-computing cluster = LocalCluster() # connect client to your cluster client = Client(cluster) # Monitor your computation with the Dask dashboard print(client.dashboard_link) ############################################################################### # Run parallel computation using dask.distributed ############################################################################### import time import joblib def long_running_function(i): time.sleep(0.1) return i ############################################################################### # The verbose messages below show that the backend is indeed the # dask.distributed one with joblib.parallel_config(backend="dask"): joblib.Parallel(verbose=100)( joblib.delayed(long_running_function)(i) for i in range(10) ) ############################################################################### joblib-1.3.2/examples/parallel_generator.py000066400000000000000000000142361446465525000210310ustar00rootroot00000000000000""" ======================================== Returning a generator in joblib.Parallel ======================================== This example illustrates memory optimization enabled by using :class:`joblib.Parallel` to get a generator on the outputs of parallel jobs. We first create tasks that return results with large memory footprints. If we call :class:`~joblib.Parallel` for several of these tasks directly, we observe a high memory usage, as all the results are held in RAM before being processed Using ``return_as='generator'`` allows to progressively consume the outputs as they arrive and keeps the memory at an acceptable level. In this case, the output of the `Parallel` call is a generator that yields the results in the order the tasks have been submitted with. Future releases are also planned to support the ``return_as="unordered_generator"`` parameter to have the generator yield results as soon as available. """ ############################################################################## # ``MemoryMonitor`` helper ############################################################################## ############################################################################## # The following class is an helper to monitor the memory of the process and its # children in another thread, so we can display it afterward. # # We will use ``psutil`` to monitor the memory usage in the code. Make sure it # is installed with ``pip install psutil`` for this example. import time from psutil import Process from threading import Thread class MemoryMonitor(Thread): """Monitor the memory usage in MB in a separate thread. Note that this class is good enough to highlight the memory profile of Parallel in this example, but is not a general purpose profiler fit for all cases. """ def __init__(self): super().__init__() self.stop = False self.memory_buffer = [] self.start() def get_memory(self): "Get memory of a process and its children." p = Process() memory = p.memory_info().rss for c in p.children(): memory += c.memory_info().rss return memory def run(self): memory_start = self.get_memory() while not self.stop: self.memory_buffer.append(self.get_memory() - memory_start) time.sleep(0.2) def join(self): self.stop = True super().join() ############################################################################## # Save memory by consuming the outputs of the tasks as fast as possible ############################################################################## ############################################################################## # We create a task whose output takes about 15MB of RAM. # import numpy as np def return_big_object(i): time.sleep(.1) return i * np.ones((10000, 200), dtype=np.float64) ############################################################################## # We create a reduce step. The input will be a generator on big objects # generated in parallel by several instances of ``return_big_object``. def accumulator_sum(generator): result = 0 for value in generator: result += value print(".", end="", flush=True) print("") return result ############################################################################## # We process many of the tasks in parallel. If ``return_as="list"`` (default), # we should expect a usage of more than 2GB in RAM. Indeed, all the results # are computed and stored in ``res`` before being processed by # `accumulator_sum` and collected by the gc. from joblib import Parallel, delayed monitor = MemoryMonitor() print("Running tasks with return_as='list'...") res = Parallel(n_jobs=2, return_as="list")( delayed(return_big_object)(i) for i in range(150) ) print("Accumulate results:", end='') res = accumulator_sum(res) print('All tasks completed and reduced successfully.') # Report memory usage del res # we clean the result to avoid memory border effects monitor.join() peak = max(monitor.memory_buffer) / 1e9 print(f"Peak memory usage: {peak:.2f}GB") ############################################################################## # If we use ``return_as="generator"``, ``res`` is simply a generator on the # results that are ready. Here we consume the results as soon as they arrive # with the ``accumulator_sum`` and once they have been used, they are collected # by the gc. The memory footprint is thus reduced, typically around 300MB. monitor_gen = MemoryMonitor() print("Create result generator with return_as='generator'...") res = Parallel(n_jobs=2, return_as="generator")( delayed(return_big_object)(i) for i in range(150) ) print("Accumulate results:", end='') res = accumulator_sum(res) print('All tasks completed and reduced successfully.') # Report memory usage del res # we clean the result to avoid memory border effects monitor_gen.join() peak = max(monitor_gen.memory_buffer) / 1e6 print(f"Peak memory usage: {peak:.2f}MB") ############################################################################## # We can then report the memory usage accross time of the two runs using the # MemoryMonitor. # # In the first case, as the results accumulate in ``res``, the memory grows # linearly and it is freed once the ``accumulator_sum`` function finishes. # # In the second case, the results are processed by the accumulator as soon as # they arrive, and the memory does not need to be able to contain all # the results. import matplotlib.pyplot as plt plt.semilogy( np.maximum.accumulate(monitor.memory_buffer), label='return_as="list"' ) plt.semilogy( np.maximum.accumulate(monitor_gen.memory_buffer), label='return_as="generator"' ) plt.xlabel("Time") plt.xticks([], []) plt.ylabel("Memory usage") plt.yticks([1e7, 1e8, 1e9], ['10MB', '100MB', '1GB']) plt.legend() plt.show() ############################################################################## # It is important to note that with ``return_as="generator"``, the results are # still accumulated in RAM after computation. But as we asynchronously process # them, they can be freed sooner. However, if the generator is not consumed # the memory still grows linearly. joblib-1.3.2/examples/parallel_memmap.py000066400000000000000000000127241446465525000203170ustar00rootroot00000000000000""" =============================== NumPy memmap in joblib.Parallel =============================== This example illustrates some features enabled by using a memory map (:class:`numpy.memmap`) within :class:`joblib.Parallel`. First, we show that dumping a huge data array ahead of passing it to :class:`joblib.Parallel` speeds up computation. Then, we show the possibility to provide write access to original data. """ ############################################################################## # Speed up processing of a large data array ############################################################################## # # We create a large data array for which the average is computed for several # slices. import numpy as np data = np.random.random((int(1e7),)) window_size = int(5e5) slices = [slice(start, start + window_size) for start in range(0, data.size - window_size, int(1e5))] ############################################################################### # The ``slow_mean`` function introduces a :func:`time.sleep` call to simulate a # more expensive computation cost for which parallel computing is beneficial. # Parallel may not be beneficial for very fast operation, due to extra overhead # (workers creations, communication, etc.). import time def slow_mean(data, sl): """Simulate a time consuming processing.""" time.sleep(0.01) return data[sl].mean() ############################################################################### # First, we will evaluate the sequential computing on our problem. tic = time.time() results = [slow_mean(data, sl) for sl in slices] toc = time.time() print('\nElapsed time computing the average of couple of slices {:.2f} s' .format(toc - tic)) ############################################################################### # :class:`joblib.Parallel` is used to compute in parallel the average of all # slices using 2 workers. from joblib import Parallel, delayed tic = time.time() results = Parallel(n_jobs=2)(delayed(slow_mean)(data, sl) for sl in slices) toc = time.time() print('\nElapsed time computing the average of couple of slices {:.2f} s' .format(toc - tic)) ############################################################################### # Parallel processing is already faster than the sequential processing. It is # also possible to remove a bit of overhead by dumping the ``data`` array to a # memmap and pass the memmap to :class:`joblib.Parallel`. import os from joblib import dump, load folder = './joblib_memmap' try: os.mkdir(folder) except FileExistsError: pass data_filename_memmap = os.path.join(folder, 'data_memmap') dump(data, data_filename_memmap) data = load(data_filename_memmap, mmap_mode='r') tic = time.time() results = Parallel(n_jobs=2)(delayed(slow_mean)(data, sl) for sl in slices) toc = time.time() print('\nElapsed time computing the average of couple of slices {:.2f} s\n' .format(toc - tic)) ############################################################################### # Therefore, dumping large ``data`` array ahead of calling # :class:`joblib.Parallel` can speed up the processing by removing some # overhead. ############################################################################### # Writable memmap for shared memory :class:`joblib.Parallel` ############################################################################### # # ``slow_mean_write_output`` will compute the mean for some given slices as in # the previous example. However, the resulting mean will be directly written on # the output array. def slow_mean_write_output(data, sl, output, idx): """Simulate a time consuming processing.""" time.sleep(0.005) res_ = data[sl].mean() print("[Worker %d] Mean for slice %d is %f" % (os.getpid(), idx, res_)) output[idx] = res_ ############################################################################### # Prepare the folder where the memmap will be dumped. output_filename_memmap = os.path.join(folder, 'output_memmap') ############################################################################### # Pre-allocate a writable shared memory map as a container for the results of # the parallel computation. output = np.memmap(output_filename_memmap, dtype=data.dtype, shape=len(slices), mode='w+') ############################################################################### # ``data`` is replaced by its memory mapped version. Note that the buffer has # already been dumped in the previous section. data = load(data_filename_memmap, mmap_mode='r') ############################################################################### # Fork the worker processes to perform computation concurrently Parallel(n_jobs=2)(delayed(slow_mean_write_output)(data, sl, output, idx) for idx, sl in enumerate(slices)) ############################################################################### # Compare the results from the output buffer with the expected results print("\nExpected means computed in the parent process:\n {}" .format(np.array(results))) print("\nActual means computed by the worker processes:\n {}" .format(output)) ############################################################################### # Clean-up the memmap ############################################################################### # # Remove the different memmap that we created. It might fail in Windows due # to file permissions. import shutil try: shutil.rmtree(folder) except: # noqa print('Could not clean-up automatically.') joblib-1.3.2/examples/parallel_random_state.py000066400000000000000000000110511446465525000215130ustar00rootroot00000000000000# %% """ =================================== Random state within joblib.Parallel =================================== Randomness is affected by parallel execution differently by the different backends. In particular, when using multiple processes, the random sequence can be the same in all processes. This example illustrates the problem and shows how to work around it. """ import numpy as np from joblib import Parallel, delayed # %% # A utility function for the example def print_vector(vector, backend): """Helper function to print the generated vector with a given backend.""" print('\nThe different generated vectors using the {} backend are:\n {}' .format(backend, np.array(vector))) # %% # Sequential behavior ##################### # # ``stochastic_function`` will generate five random integers. When # calling the function several times, we are expecting to obtain # different vectors. For instance, we will call the function five times # in a sequential manner, we can check that the generated vectors are all # different. def stochastic_function(max_value): """Randomly generate integer up to a maximum value.""" return np.random.randint(max_value, size=5) n_vectors = 5 random_vector = [stochastic_function(10) for _ in range(n_vectors)] print('\nThe different generated vectors in a sequential manner are:\n {}' .format(np.array(random_vector))) # %% # Parallel behavior ################### # # Joblib provides three different backends: loky (default), threading, and # multiprocessing. backend = 'loky' random_vector = Parallel(n_jobs=2, backend=backend)(delayed( stochastic_function)(10) for _ in range(n_vectors)) print_vector(random_vector, backend) ############################################################################### backend = 'threading' random_vector = Parallel(n_jobs=2, backend=backend)(delayed( stochastic_function)(10) for _ in range(n_vectors)) print_vector(random_vector, backend) # %% # Loky and the threading backends behave exactly as in the sequential case and # do not require more care. However, this is not the case regarding the # multiprocessing backend with the "fork" or "forkserver" start method because # the state of the global numpy random stated will be exactly duplicated # in all the workers # # Note: on platforms for which the default start method is "spawn", we do not # have this problem but we cannot use this in a Python script without # using the if __name__ == "__main__" construct. So let's end this example # early if that's the case: import multiprocessing as mp if mp.get_start_method() != "spawn": backend = 'multiprocessing' random_vector = Parallel(n_jobs=2, backend=backend)(delayed( stochastic_function)(10) for _ in range(n_vectors)) print_vector(random_vector, backend) # %% # Some of the generated vectors are exactly the same, which can be a # problem for the application. # # Technically, the reason is that all forked Python processes share the # same exact random seed. As a result, we obtain twice the same randomly # generated vectors because we are using ``n_jobs=2``. A solution is to # set the random state within the function which is passed to # :class:`joblib.Parallel`. def stochastic_function_seeded(max_value, random_state): rng = np.random.RandomState(random_state) return rng.randint(max_value, size=5) # %% # ``stochastic_function_seeded`` accepts as argument a random seed. We can # reset this seed by passing ``None`` at every function call. In this case, we # see that the generated vectors are all different. if mp.get_start_method() != "spawn": random_vector = Parallel(n_jobs=2, backend=backend)(delayed( stochastic_function_seeded)(10, None) for _ in range(n_vectors)) print_vector(random_vector, backend) # %% # Fixing the random state to obtain deterministic results ######################################################### # # The pattern of ``stochastic_function_seeded`` has another advantage: it # allows to control the random_state by passing a known seed. So for instance, # we can replicate the same generation of vectors by passing a fixed state as # follows. if mp.get_start_method() != "spawn": random_state = np.random.randint(np.iinfo(np.int32).max, size=n_vectors) random_vector = Parallel(n_jobs=2, backend=backend)(delayed( stochastic_function_seeded)(10, rng) for rng in random_state) print_vector(random_vector, backend) random_vector = Parallel(n_jobs=2, backend=backend)(delayed( stochastic_function_seeded)(10, rng) for rng in random_state) print_vector(random_vector, backend) joblib-1.3.2/examples/serialization_and_wrappers.py000066400000000000000000000131331446465525000226040ustar00rootroot00000000000000# -*- coding: utf-8 -*- """ Serialization of un-picklable objects ===================================== This example highlights the options for tempering with joblib serialization process. """ # Code source: Thomas Moreau # License: BSD 3 clause import sys import time import traceback from joblib.externals.loky import set_loky_pickler from joblib import parallel_config from joblib import Parallel, delayed from joblib import wrap_non_picklable_objects ############################################################################### # First, define functions which cannot be pickled with the standard ``pickle`` # protocol. They cannot be serialized with ``pickle`` because they are defined # in the ``__main__`` module. They can however be serialized with # ``cloudpickle``. With the default behavior, ``loky`` is to use # ``cloudpickle`` to serialize the objects that are sent to the workers. # def func_async(i, *args): return 2 * i print(Parallel(n_jobs=2)(delayed(func_async)(21) for _ in range(1))[0]) ############################################################################### # For most use-cases, using ``cloudpickle`` is efficient enough. However, this # solution can be very slow to serialize large python objects, such as dict or # list, compared to the standard ``pickle`` serialization. # def func_async(i, *args): return 2 * i # We have to pass an extra argument with a large list (or another large python # object). large_list = list(range(1000000)) t_start = time.time() Parallel(n_jobs=2)(delayed(func_async)(21, large_list) for _ in range(1)) print("With loky backend and cloudpickle serialization: {:.3f}s" .format(time.time() - t_start)) ############################################################################### # If you are on a UNIX system, it is possible to fallback to the old # ``multiprocessing`` backend, which can pickle interactively defined functions # with the default pickle module, which is faster for such large objects. # import multiprocessing as mp if mp.get_start_method() != "spawn": def func_async(i, *args): return 2 * i with parallel_config('multiprocessing'): t_start = time.time() Parallel(n_jobs=2)( delayed(func_async)(21, large_list) for _ in range(1)) print("With multiprocessing backend and pickle serialization: {:.3f}s" .format(time.time() - t_start)) ############################################################################### # However, using ``fork`` to start new processes can cause violation of the # POSIX specification and can have bad interaction with compiled extensions # that use ``openmp``. Also, it is not possible to start processes with # ``fork`` on windows where only ``spawn`` is available. The ``loky`` backend # has been developed to mitigate these issues. # # To have fast pickling with ``loky``, it is possible to rely on ``pickle`` to # serialize all communications between the main process and the workers with # the ``loky`` backend. This can be done by setting the environment variable # ``LOKY_PICKLER=pickle`` before the script is launched. Here we use an # internal programmatic switch ``loky.set_loky_pickler`` for demonstration # purposes but it has the same effect as setting ``LOKY_PICKLER``. Note that # this switch should not be used as it has some side effects with the workers. # # Now set the `loky_pickler` to use the pickle serialization from stdlib. Here, # we do not pass the desired function ``func_async`` as it is not picklable # but it is replaced by ``id`` for demonstration purposes. set_loky_pickler('pickle') t_start = time.time() Parallel(n_jobs=2)(delayed(id)(large_list) for _ in range(1)) print("With pickle serialization: {:.3f}s".format(time.time() - t_start)) ############################################################################### # However, the function and objects defined in ``__main__`` are not # serializable anymore using ``pickle`` and it is not possible to call # ``func_async`` using this pickler. # def func_async(i, *args): return 2 * i try: Parallel(n_jobs=2)(delayed(func_async)(21, large_list) for _ in range(1)) except Exception: traceback.print_exc(file=sys.stdout) ############################################################################### # To have both fast pickling, safe process creation and serialization of # interactive functions, ``joblib`` provides a wrapper function # :func:`~joblib.wrap_non_picklable_objects` to wrap the non-picklable function # and indicate to the serialization process that this specific function should # be serialized using ``cloudpickle``. This changes the serialization behavior # only for this function and keeps using ``pickle`` for all other objects. The # drawback of this solution is that it modifies the object. This should not # cause many issues with functions but can have side effects with object # instances. # @delayed @wrap_non_picklable_objects def func_async_wrapped(i, *args): return 2 * i t_start = time.time() Parallel(n_jobs=2)(func_async_wrapped(21, large_list) for _ in range(1)) print("With pickle from stdlib and wrapper: {:.3f}s" .format(time.time() - t_start)) ############################################################################### # The same wrapper can also be used for non-picklable classes. Note that the # side effects of ``wrap_non_picklable_objects`` on objects can break magic # methods such as ``__add__`` and can mess up the ``isinstance`` and # ``issubclass`` functions. Some improvements will be considered if use-cases # are reported. # # Reset the loky_pickler to avoid border effects with other examples in # sphinx-gallery. set_loky_pickler() joblib-1.3.2/joblib/000077500000000000000000000000001446465525000142325ustar00rootroot00000000000000joblib-1.3.2/joblib/__init__.py000066400000000000000000000120141446465525000163410ustar00rootroot00000000000000"""Joblib is a set of tools to provide **lightweight pipelining in Python**. In particular: 1. transparent disk-caching of functions and lazy re-evaluation (memoize pattern) 2. easy simple parallel computing Joblib is optimized to be **fast** and **robust** on large data in particular and has specific optimizations for `numpy` arrays. It is **BSD-licensed**. ==================== =============================================== **Documentation:** https://joblib.readthedocs.io **Download:** https://pypi.python.org/pypi/joblib#downloads **Source code:** https://github.com/joblib/joblib **Report issues:** https://github.com/joblib/joblib/issues ==================== =============================================== Vision -------- The vision is to provide tools to easily achieve better performance and reproducibility when working with long running jobs. * **Avoid computing the same thing twice**: code is often rerun again and again, for instance when prototyping computational-heavy jobs (as in scientific development), but hand-crafted solutions to alleviate this issue are error-prone and often lead to unreproducible results. * **Persist to disk transparently**: efficiently persisting arbitrary objects containing large data is hard. Using joblib's caching mechanism avoids hand-written persistence and implicitly links the file on disk to the execution context of the original Python object. As a result, joblib's persistence is good for resuming an application status or computational job, eg after a crash. Joblib addresses these problems while **leaving your code and your flow control as unmodified as possible** (no framework, no new paradigms). Main features ------------------ 1) **Transparent and fast disk-caching of output value:** a memoize or make-like functionality for Python functions that works well for arbitrary Python objects, including very large numpy arrays. Separate persistence and flow-execution logic from domain logic or algorithmic code by writing the operations as a set of steps with well-defined inputs and outputs: Python functions. Joblib can save their computation to disk and rerun it only if necessary:: >>> from joblib import Memory >>> cachedir = 'your_cache_dir_goes_here' >>> mem = Memory(cachedir) >>> import numpy as np >>> a = np.vander(np.arange(3)).astype(float) >>> square = mem.cache(np.square) >>> b = square(a) # doctest: +ELLIPSIS ______________________________________________________________________... [Memory] Calling square... square(array([[0., 0., 1.], [1., 1., 1.], [4., 2., 1.]])) _________________________________________________...square - ...s, 0.0min >>> c = square(a) >>> # The above call did not trigger an evaluation 2) **Embarrassingly parallel helper:** to make it easy to write readable parallel code and debug it quickly:: >>> from joblib import Parallel, delayed >>> from math import sqrt >>> Parallel(n_jobs=1)(delayed(sqrt)(i**2) for i in range(10)) [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] 3) **Fast compressed Persistence**: a replacement for pickle to work efficiently on Python objects containing large data ( *joblib.dump* & *joblib.load* ). .. >>> import shutil ; shutil.rmtree(cachedir) """ # PEP0440 compatible formatted version, see: # https://www.python.org/dev/peps/pep-0440/ # # Generic release markers: # X.Y # X.Y.Z # For bugfix releases # # Admissible pre-release markers: # X.YaN # Alpha release # X.YbN # Beta release # X.YrcN # Release Candidate # X.Y # Final release # # Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer. # 'X.Y.dev0' is the canonical version of 'X.Y.dev' # __version__ = '1.3.2' import os from .memory import Memory from .memory import MemorizedResult from .memory import register_store_backend from .memory import expires_after from .logger import PrintTime from .logger import Logger from .hashing import hash from .numpy_pickle import dump from .numpy_pickle import load from .compressor import register_compressor from .parallel import Parallel from .parallel import delayed from .parallel import cpu_count from .parallel import register_parallel_backend from .parallel import parallel_backend from .parallel import parallel_config from .parallel import effective_n_jobs from ._cloudpickle_wrapper import wrap_non_picklable_objects __all__ = ['Memory', 'MemorizedResult', 'PrintTime', 'Logger', 'hash', 'dump', 'load', 'Parallel', 'delayed', 'cpu_count', 'effective_n_jobs', 'register_parallel_backend', 'parallel_backend', 'expires_after', 'register_store_backend', 'register_compressor', 'wrap_non_picklable_objects', 'parallel_config'] # Workaround issue discovered in intel-openmp 2019.5: # https://github.com/ContinuumIO/anaconda-issues/issues/11294 os.environ.setdefault("KMP_INIT_AT_FORK", "FALSE") joblib-1.3.2/joblib/_cloudpickle_wrapper.py000066400000000000000000000006411446465525000210020ustar00rootroot00000000000000""" Small shim of loky's cloudpickle_wrapper to avoid failure when multiprocessing is not available. """ from ._multiprocessing_helpers import mp def _my_wrap_non_picklable_objects(obj, keep_wrapper=True): return obj if mp is not None: from .externals.loky import wrap_non_picklable_objects else: wrap_non_picklable_objects = _my_wrap_non_picklable_objects __all__ = ["wrap_non_picklable_objects"] joblib-1.3.2/joblib/_dask.py000066400000000000000000000313261446465525000156720ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import asyncio import concurrent.futures import contextlib import time from uuid import uuid4 import weakref from .parallel import parallel_config from .parallel import AutoBatchingMixin, ParallelBackendBase try: import dask import distributed except ImportError: dask = None distributed = None if dask is not None and distributed is not None: from dask.utils import funcname from dask.sizeof import sizeof from dask.distributed import ( Client, as_completed, get_client, secede, rejoin, ) from distributed.utils import thread_state try: # asyncio.TimeoutError, Python3-only error thrown by recent versions of # distributed from distributed.utils import TimeoutError as _TimeoutError except ImportError: from tornado.gen import TimeoutError as _TimeoutError def is_weakrefable(obj): try: weakref.ref(obj) return True except TypeError: return False class _WeakKeyDictionary: """A variant of weakref.WeakKeyDictionary for unhashable objects. This datastructure is used to store futures for broadcasted data objects such as large numpy arrays or pandas dataframes that are not hashable and therefore cannot be used as keys of traditional python dicts. Furthermore using a dict with id(array) as key is not safe because the Python is likely to reuse id of recently collected arrays. """ def __init__(self): self._data = {} def __getitem__(self, obj): ref, val = self._data[id(obj)] if ref() is not obj: # In case of a race condition with on_destroy. raise KeyError(obj) return val def __setitem__(self, obj, value): key = id(obj) try: ref, _ = self._data[key] if ref() is not obj: # In case of race condition with on_destroy. raise KeyError(obj) except KeyError: # Insert the new entry in the mapping along with a weakref # callback to automatically delete the entry from the mapping # as soon as the object used as key is garbage collected. def on_destroy(_): del self._data[key] ref = weakref.ref(obj, on_destroy) self._data[key] = ref, value def __len__(self): return len(self._data) def clear(self): self._data.clear() def _funcname(x): try: if isinstance(x, list): x = x[0][0] except Exception: pass return funcname(x) def _make_tasks_summary(tasks): """Summarize of list of (func, args, kwargs) function calls""" unique_funcs = {func for func, args, kwargs in tasks} if len(unique_funcs) == 1: mixed = False else: mixed = True return len(tasks), mixed, _funcname(tasks) class Batch: """dask-compatible wrapper that executes a batch of tasks""" def __init__(self, tasks): # collect some metadata from the tasks to ease Batch calls # introspection when debugging self._num_tasks, self._mixed, self._funcname = _make_tasks_summary( tasks ) def __call__(self, tasks=None): results = [] with parallel_config(backend='dask'): for func, args, kwargs in tasks: results.append(func(*args, **kwargs)) return results def __repr__(self): descr = f"batch_of_{self._funcname}_{self._num_tasks}_calls" if self._mixed: descr = "mixed_" + descr return descr def _joblib_probe_task(): # Noop used by the joblib connector to probe when workers are ready. pass class DaskDistributedBackend(AutoBatchingMixin, ParallelBackendBase): MIN_IDEAL_BATCH_DURATION = 0.2 MAX_IDEAL_BATCH_DURATION = 1.0 supports_timeout = True default_n_jobs = -1 def __init__(self, scheduler_host=None, scatter=None, client=None, loop=None, wait_for_workers_timeout=10, **submit_kwargs): super().__init__() if distributed is None: msg = ("You are trying to use 'dask' as a joblib parallel backend " "but dask is not installed. Please install dask " "to fix this error.") raise ValueError(msg) if client is None: if scheduler_host: client = Client(scheduler_host, loop=loop, set_as_default=False) else: try: client = get_client() except ValueError as e: msg = ("To use Joblib with Dask first create a Dask Client" "\n\n" " from dask.distributed import Client\n" " client = Client()\n" "or\n" " client = Client('scheduler-address:8786')") raise ValueError(msg) from e self.client = client if scatter is not None and not isinstance(scatter, (list, tuple)): raise TypeError("scatter must be a list/tuple, got " "`%s`" % type(scatter).__name__) if scatter is not None and len(scatter) > 0: # Keep a reference to the scattered data to keep the ids the same self._scatter = list(scatter) scattered = self.client.scatter(scatter, broadcast=True) self.data_futures = {id(x): f for x, f in zip(scatter, scattered)} else: self._scatter = [] self.data_futures = {} self.wait_for_workers_timeout = wait_for_workers_timeout self.submit_kwargs = submit_kwargs self.waiting_futures = as_completed( [], loop=client.loop, with_results=True, raise_errors=False ) self._results = {} self._callbacks = {} async def _collect(self): while self._continue: async for future, result in self.waiting_futures: cf_future = self._results.pop(future) callback = self._callbacks.pop(future) if future.status == "error": typ, exc, tb = result cf_future.set_exception(exc) else: cf_future.set_result(result) callback(result) await asyncio.sleep(0.01) def __reduce__(self): return (DaskDistributedBackend, ()) def get_nested_backend(self): return DaskDistributedBackend(client=self.client), -1 def configure(self, n_jobs=1, parallel=None, **backend_args): self.parallel = parallel return self.effective_n_jobs(n_jobs) def start_call(self): self._continue = True self.client.loop.add_callback(self._collect) self.call_data_futures = _WeakKeyDictionary() def stop_call(self): # The explicit call to clear is required to break a cycling reference # to the futures. self._continue = False # wait for the future collection routine (self._backend._collect) to # finish in order to limit asyncio warnings due to aborting _collect # during a following backend termination call time.sleep(0.01) self.call_data_futures.clear() def effective_n_jobs(self, n_jobs): effective_n_jobs = sum(self.client.ncores().values()) if effective_n_jobs != 0 or not self.wait_for_workers_timeout: return effective_n_jobs # If there is no worker, schedule a probe task to wait for the workers # to come up and be available. If the dask cluster is in adaptive mode # task might cause the cluster to provision some workers. try: self.client.submit(_joblib_probe_task).result( timeout=self.wait_for_workers_timeout ) except _TimeoutError as e: error_msg = ( "DaskDistributedBackend has no worker after {} seconds. " "Make sure that workers are started and can properly connect " "to the scheduler and increase the joblib/dask connection " "timeout with:\n\n" "parallel_config(backend='dask', wait_for_workers_timeout={})" ).format(self.wait_for_workers_timeout, max(10, 2 * self.wait_for_workers_timeout)) raise TimeoutError(error_msg) from e return sum(self.client.ncores().values()) async def _to_func_args(self, func): itemgetters = dict() # Futures that are dynamically generated during a single call to # Parallel.__call__. call_data_futures = getattr(self, 'call_data_futures', None) async def maybe_to_futures(args): out = [] for arg in args: arg_id = id(arg) if arg_id in itemgetters: out.append(itemgetters[arg_id]) continue f = self.data_futures.get(arg_id, None) if f is None and call_data_futures is not None: try: f = await call_data_futures[arg] except KeyError: pass if f is None: if is_weakrefable(arg) and sizeof(arg) > 1e3: # Automatically scatter large objects to some of # the workers to avoid duplicated data transfers. # Rely on automated inter-worker data stealing if # more workers need to reuse this data # concurrently. # set hash=False - nested scatter calls (i.e # calling client.scatter inside a dask worker) # using hash=True often raise CancelledError, # see dask/distributed#3703 _coro = self.client.scatter( arg, asynchronous=True, hash=False ) # Centralize the scattering of identical arguments # between concurrent apply_async callbacks by # exposing the running coroutine in # call_data_futures before it completes. t = asyncio.Task(_coro) call_data_futures[arg] = t f = await t if f is not None: out.append(f) else: out.append(arg) return out tasks = [] for f, args, kwargs in func.items: args = list(await maybe_to_futures(args)) kwargs = dict(zip(kwargs.keys(), await maybe_to_futures(kwargs.values()))) tasks.append((f, args, kwargs)) return (Batch(tasks), tasks) def apply_async(self, func, callback=None): cf_future = concurrent.futures.Future() cf_future.get = cf_future.result # achieve AsyncResult API async def f(func, callback): batch, tasks = await self._to_func_args(func) key = f'{repr(batch)}-{uuid4().hex}' dask_future = self.client.submit( batch, tasks=tasks, key=key, **self.submit_kwargs ) self.waiting_futures.add(dask_future) self._callbacks[dask_future] = callback self._results[dask_future] = cf_future self.client.loop.add_callback(f, func, callback) return cf_future def abort_everything(self, ensure_ready=True): """ Tell the client to cancel any task submitted via this instance joblib.Parallel will never access those results """ with self.waiting_futures.lock: self.waiting_futures.futures.clear() while not self.waiting_futures.queue.empty(): self.waiting_futures.queue.get() @contextlib.contextmanager def retrieval_context(self): """Override ParallelBackendBase.retrieval_context to avoid deadlocks. This removes thread from the worker's thread pool (using 'secede'). Seceding avoids deadlock in nested parallelism settings. """ # See 'joblib.Parallel.__call__' and 'joblib.Parallel.retrieve' for how # this is used. if hasattr(thread_state, 'execution_state'): # we are in a worker. Secede to avoid deadlock. secede() yield if hasattr(thread_state, 'execution_state'): rejoin() joblib-1.3.2/joblib/_memmapping_reducer.py000066400000000000000000000664041446465525000206200ustar00rootroot00000000000000""" Reducer using memory mapping for numpy arrays """ # Author: Thomas Moreau # Copyright: 2017, Thomas Moreau # License: BSD 3 clause from mmap import mmap import errno import os import stat import threading import atexit import tempfile import time import warnings import weakref from uuid import uuid4 from multiprocessing import util from pickle import whichmodule, loads, dumps, HIGHEST_PROTOCOL, PicklingError try: WindowsError except NameError: WindowsError = type(None) try: import numpy as np from numpy.lib.stride_tricks import as_strided except ImportError: np = None from .numpy_pickle import dump, load, load_temporary_memmap from .backports import make_memmap from .disk import delete_folder from .externals.loky.backend import resource_tracker # Some system have a ramdisk mounted by default, we can use it instead of /tmp # as the default folder to dump big arrays to share with subprocesses. SYSTEM_SHARED_MEM_FS = '/dev/shm' # Minimal number of bytes available on SYSTEM_SHARED_MEM_FS to consider using # it as the default folder to dump big arrays to share with subprocesses. SYSTEM_SHARED_MEM_FS_MIN_SIZE = int(2e9) # Folder and file permissions to chmod temporary files generated by the # memmapping pool. Only the owner of the Python process can access the # temporary files and folder. FOLDER_PERMISSIONS = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR FILE_PERMISSIONS = stat.S_IRUSR | stat.S_IWUSR # Set used in joblib workers, referencing the filenames of temporary memmaps # created by joblib to speed up data communication. In child processes, we add # a finalizer to these memmaps that sends a maybe_unlink call to the # resource_tracker, in order to free main memory as fast as possible. JOBLIB_MMAPS = set() def _log_and_unlink(filename): from .externals.loky.backend.resource_tracker import _resource_tracker util.debug( "[FINALIZER CALL] object mapping to {} about to be deleted," " decrementing the refcount of the file (pid: {})".format( os.path.basename(filename), os.getpid())) _resource_tracker.maybe_unlink(filename, "file") def add_maybe_unlink_finalizer(memmap): util.debug( "[FINALIZER ADD] adding finalizer to {} (id {}, filename {}, pid {})" "".format(type(memmap), id(memmap), os.path.basename(memmap.filename), os.getpid())) weakref.finalize(memmap, _log_and_unlink, memmap.filename) def unlink_file(filename): """Wrapper around os.unlink with a retry mechanism. The retry mechanism has been implemented primarily to overcome a race condition happening during the finalizer of a np.memmap: when a process holding the last reference to a mmap-backed np.memmap/np.array is about to delete this array (and close the reference), it sends a maybe_unlink request to the resource_tracker. This request can be processed faster than it takes for the last reference of the memmap to be closed, yielding (on Windows) a PermissionError in the resource_tracker loop. """ NUM_RETRIES = 10 for retry_no in range(1, NUM_RETRIES + 1): try: os.unlink(filename) break except PermissionError: util.debug( '[ResourceTracker] tried to unlink {}, got ' 'PermissionError'.format(filename) ) if retry_no == NUM_RETRIES: raise else: time.sleep(.2) except FileNotFoundError: # In case of a race condition when deleting the temporary folder, # avoid noisy FileNotFoundError exception in the resource tracker. pass resource_tracker._CLEANUP_FUNCS['file'] = unlink_file class _WeakArrayKeyMap: """A variant of weakref.WeakKeyDictionary for unhashable numpy arrays. This datastructure will be used with numpy arrays as obj keys, therefore we do not use the __get__ / __set__ methods to avoid any conflict with the numpy fancy indexing syntax. """ def __init__(self): self._data = {} def get(self, obj): ref, val = self._data[id(obj)] if ref() is not obj: # In case of race condition with on_destroy: could never be # triggered by the joblib tests with CPython. raise KeyError(obj) return val def set(self, obj, value): key = id(obj) try: ref, _ = self._data[key] if ref() is not obj: # In case of race condition with on_destroy: could never be # triggered by the joblib tests with CPython. raise KeyError(obj) except KeyError: # Insert the new entry in the mapping along with a weakref # callback to automatically delete the entry from the mapping # as soon as the object used as key is garbage collected. def on_destroy(_): del self._data[key] ref = weakref.ref(obj, on_destroy) self._data[key] = ref, value def __getstate__(self): raise PicklingError("_WeakArrayKeyMap is not pickleable") ############################################################################### # Support for efficient transient pickling of numpy data structures def _get_backing_memmap(a): """Recursively look up the original np.memmap instance base if any.""" b = getattr(a, 'base', None) if b is None: # TODO: check scipy sparse datastructure if scipy is installed # a nor its descendants do not have a memmap base return None elif isinstance(b, mmap): # a is already a real memmap instance. return a else: # Recursive exploration of the base ancestry return _get_backing_memmap(b) def _get_temp_dir(pool_folder_name, temp_folder=None): """Get the full path to a subfolder inside the temporary folder. Parameters ---------- pool_folder_name : str Sub-folder name used for the serialization of a pool instance. temp_folder: str, optional Folder to be used by the pool for memmapping large arrays for sharing memory with worker processes. If None, this will try in order: - a folder pointed by the JOBLIB_TEMP_FOLDER environment variable, - /dev/shm if the folder exists and is writable: this is a RAMdisk filesystem available by default on modern Linux distributions, - the default system temporary folder that can be overridden with TMP, TMPDIR or TEMP environment variables, typically /tmp under Unix operating systems. Returns ------- pool_folder : str full path to the temporary folder use_shared_mem : bool whether the temporary folder is written to the system shared memory folder or some other temporary folder. """ use_shared_mem = False if temp_folder is None: temp_folder = os.environ.get('JOBLIB_TEMP_FOLDER', None) if temp_folder is None: if os.path.exists(SYSTEM_SHARED_MEM_FS) and hasattr(os, 'statvfs'): try: shm_stats = os.statvfs(SYSTEM_SHARED_MEM_FS) available_nbytes = shm_stats.f_bsize * shm_stats.f_bavail if available_nbytes > SYSTEM_SHARED_MEM_FS_MIN_SIZE: # Try to see if we have write access to the shared mem # folder only if it is reasonably large (that is 2GB or # more). temp_folder = SYSTEM_SHARED_MEM_FS pool_folder = os.path.join(temp_folder, pool_folder_name) if not os.path.exists(pool_folder): os.makedirs(pool_folder) use_shared_mem = True except (IOError, OSError): # Missing rights in the /dev/shm partition, fallback to regular # temp folder. temp_folder = None if temp_folder is None: # Fallback to the default tmp folder, typically /tmp temp_folder = tempfile.gettempdir() temp_folder = os.path.abspath(os.path.expanduser(temp_folder)) pool_folder = os.path.join(temp_folder, pool_folder_name) return pool_folder, use_shared_mem def has_shareable_memory(a): """Return True if a is backed by some mmap buffer directly or not.""" return _get_backing_memmap(a) is not None def _strided_from_memmap(filename, dtype, mode, offset, order, shape, strides, total_buffer_len, unlink_on_gc_collect): """Reconstruct an array view on a memory mapped file.""" if mode == 'w+': # Do not zero the original data when unpickling mode = 'r+' if strides is None: # Simple, contiguous memmap return make_memmap( filename, dtype=dtype, shape=shape, mode=mode, offset=offset, order=order, unlink_on_gc_collect=unlink_on_gc_collect ) else: # For non-contiguous data, memmap the total enclosing buffer and then # extract the non-contiguous view with the stride-tricks API base = make_memmap( filename, dtype=dtype, shape=total_buffer_len, offset=offset, mode=mode, order=order, unlink_on_gc_collect=unlink_on_gc_collect ) return as_strided(base, shape=shape, strides=strides) def _reduce_memmap_backed(a, m): """Pickling reduction for memmap backed arrays. a is expected to be an instance of np.ndarray (or np.memmap) m is expected to be an instance of np.memmap on the top of the ``base`` attribute ancestry of a. ``m.base`` should be the real python mmap object. """ # offset that comes from the striding differences between a and m util.debug('[MEMMAP REDUCE] reducing a memmap-backed array ' '(shape, {}, pid: {})'.format(a.shape, os.getpid())) a_start, a_end = np.byte_bounds(a) m_start = np.byte_bounds(m)[0] offset = a_start - m_start # offset from the backing memmap offset += m.offset if m.flags['F_CONTIGUOUS']: order = 'F' else: # The backing memmap buffer is necessarily contiguous hence C if not # Fortran order = 'C' if a.flags['F_CONTIGUOUS'] or a.flags['C_CONTIGUOUS']: # If the array is a contiguous view, no need to pass the strides strides = None total_buffer_len = None else: # Compute the total number of items to map from which the strided # view will be extracted. strides = a.strides total_buffer_len = (a_end - a_start) // a.itemsize return (_strided_from_memmap, (m.filename, a.dtype, m.mode, offset, order, a.shape, strides, total_buffer_len, False)) def reduce_array_memmap_backward(a): """reduce a np.array or a np.memmap from a child process""" m = _get_backing_memmap(a) if isinstance(m, np.memmap) and m.filename not in JOBLIB_MMAPS: # if a is backed by a memmaped file, reconstruct a using the # memmaped file. return _reduce_memmap_backed(a, m) else: # a is either a regular (not memmap-backed) numpy array, or an array # backed by a shared temporary file created by joblib. In the latter # case, in order to limit the lifespan of these temporary files, we # serialize the memmap as a regular numpy array, and decref the # file backing the memmap (done implicitly in a previously registered # finalizer, see ``unlink_on_gc_collect`` for more details) return ( loads, (dumps(np.asarray(a), protocol=HIGHEST_PROTOCOL), ) ) class ArrayMemmapForwardReducer(object): """Reducer callable to dump large arrays to memmap files. Parameters ---------- max_nbytes: int Threshold to trigger memmapping of large arrays to files created a folder. temp_folder_resolver: callable An callable in charge of resolving a temporary folder name where files for backing memmapped arrays are created. mmap_mode: 'r', 'r+' or 'c' Mode for the created memmap datastructure. See the documentation of numpy.memmap for more details. Note: 'w+' is coerced to 'r+' automatically to avoid zeroing the data on unpickling. verbose: int, optional, 0 by default If verbose > 0, memmap creations are logged. If verbose > 1, both memmap creations, reuse and array pickling are logged. prewarm: bool, optional, False by default. Force a read on newly memmapped array to make sure that OS pre-cache it memory. This can be useful to avoid concurrent disk access when the same data array is passed to different worker processes. """ def __init__(self, max_nbytes, temp_folder_resolver, mmap_mode, unlink_on_gc_collect, verbose=0, prewarm=True): self._max_nbytes = max_nbytes self._temp_folder_resolver = temp_folder_resolver self._mmap_mode = mmap_mode self.verbose = int(verbose) if prewarm == "auto": self._prewarm = not self._temp_folder.startswith( SYSTEM_SHARED_MEM_FS ) else: self._prewarm = prewarm self._prewarm = prewarm self._memmaped_arrays = _WeakArrayKeyMap() self._temporary_memmaped_filenames = set() self._unlink_on_gc_collect = unlink_on_gc_collect @property def _temp_folder(self): return self._temp_folder_resolver() def __reduce__(self): # The ArrayMemmapForwardReducer is passed to the children processes: it # needs to be pickled but the _WeakArrayKeyMap need to be skipped as # it's only guaranteed to be consistent with the parent process memory # garbage collection. # Although this reducer is pickled, it is not needed in its destination # process (child processes), as we only use this reducer to send # memmaps from the parent process to the children processes. For this # reason, we can afford skipping the resolver, (which would otherwise # be unpicklable), and pass it as None instead. args = (self._max_nbytes, None, self._mmap_mode, self._unlink_on_gc_collect) kwargs = { 'verbose': self.verbose, 'prewarm': self._prewarm, } return ArrayMemmapForwardReducer, args, kwargs def __call__(self, a): m = _get_backing_memmap(a) if m is not None and isinstance(m, np.memmap): # a is already backed by a memmap file, let's reuse it directly return _reduce_memmap_backed(a, m) if (not a.dtype.hasobject and self._max_nbytes is not None and a.nbytes > self._max_nbytes): # check that the folder exists (lazily create the pool temp folder # if required) try: os.makedirs(self._temp_folder) os.chmod(self._temp_folder, FOLDER_PERMISSIONS) except OSError as e: if e.errno != errno.EEXIST: raise e try: basename = self._memmaped_arrays.get(a) except KeyError: # Generate a new unique random filename. The process and thread # ids are only useful for debugging purpose and to make it # easier to cleanup orphaned files in case of hard process # kill (e.g. by "kill -9" or segfault). basename = "{}-{}-{}.pkl".format( os.getpid(), id(threading.current_thread()), uuid4().hex) self._memmaped_arrays.set(a, basename) filename = os.path.join(self._temp_folder, basename) # In case the same array with the same content is passed several # times to the pool subprocess children, serialize it only once is_new_memmap = filename not in self._temporary_memmaped_filenames # add the memmap to the list of temporary memmaps created by joblib self._temporary_memmaped_filenames.add(filename) if self._unlink_on_gc_collect: # Bump reference count of the memmap by 1 to account for # shared usage of the memmap by a child process. The # corresponding decref call will be executed upon calling # resource_tracker.maybe_unlink, registered as a finalizer in # the child. # the incref/decref calls here are only possible when the child # and the parent share the same resource_tracker. It is not the # case for the multiprocessing backend, but it does not matter # because unlinking a memmap from a child process is only # useful to control the memory usage of long-lasting child # processes, while the multiprocessing-based pools terminate # their workers at the end of a map() call. resource_tracker.register(filename, "file") if is_new_memmap: # Incref each temporary memmap created by joblib one extra # time. This means that these memmaps will only be deleted # once an extra maybe_unlink() is called, which is done once # all the jobs have completed (or been canceled) in the # Parallel._terminate_backend() method. resource_tracker.register(filename, "file") if not os.path.exists(filename): util.debug( "[ARRAY DUMP] Pickling new array (shape={}, dtype={}) " "creating a new memmap at {}".format( a.shape, a.dtype, filename)) for dumped_filename in dump(a, filename): os.chmod(dumped_filename, FILE_PERMISSIONS) if self._prewarm: # Warm up the data by accessing it. This operation ensures # that the disk access required to create the memmapping # file are performed in the reducing process and avoids # concurrent memmap creation in multiple children # processes. load(filename, mmap_mode=self._mmap_mode).max() else: util.debug( "[ARRAY DUMP] Pickling known array (shape={}, dtype={}) " "reusing memmap file: {}".format( a.shape, a.dtype, os.path.basename(filename))) # The worker process will use joblib.load to memmap the data return ( (load_temporary_memmap, (filename, self._mmap_mode, self._unlink_on_gc_collect)) ) else: # do not convert a into memmap, let pickler do its usual copy with # the default system pickler util.debug( '[ARRAY DUMP] Pickling array (NO MEMMAPPING) (shape={}, ' ' dtype={}).'.format(a.shape, a.dtype)) return (loads, (dumps(a, protocol=HIGHEST_PROTOCOL),)) def get_memmapping_reducers( forward_reducers=None, backward_reducers=None, temp_folder_resolver=None, max_nbytes=1e6, mmap_mode='r', verbose=0, prewarm=False, unlink_on_gc_collect=True, **kwargs): """Construct a pair of memmapping reducer linked to a tmpdir. This function manage the creation and the clean up of the temporary folders underlying the memory maps and should be use to get the reducers necessary to construct joblib pool or executor. """ if forward_reducers is None: forward_reducers = dict() if backward_reducers is None: backward_reducers = dict() if np is not None: # Register smart numpy.ndarray reducers that detects memmap backed # arrays and that is also able to dump to memmap large in-memory # arrays over the max_nbytes threshold forward_reduce_ndarray = ArrayMemmapForwardReducer( max_nbytes, temp_folder_resolver, mmap_mode, unlink_on_gc_collect, verbose, prewarm=prewarm) forward_reducers[np.ndarray] = forward_reduce_ndarray forward_reducers[np.memmap] = forward_reduce_ndarray # Communication from child process to the parent process always # pickles in-memory numpy.ndarray without dumping them as memmap # to avoid confusing the caller and make it tricky to collect the # temporary folder backward_reducers[np.ndarray] = reduce_array_memmap_backward backward_reducers[np.memmap] = reduce_array_memmap_backward return forward_reducers, backward_reducers class TemporaryResourcesManager(object): """Stateful object able to manage temporary folder and pickles It exposes: - a per-context folder name resolving API that memmap-based reducers will rely on to know where to pickle the temporary memmaps - a temporary file/folder management API that internally uses the resource_tracker. """ def __init__(self, temp_folder_root=None, context_id=None): self._current_temp_folder = None self._temp_folder_root = temp_folder_root self._use_shared_mem = None self._cached_temp_folders = dict() self._id = uuid4().hex self._finalizers = {} if context_id is None: # It would be safer to not assign a default context id (less silent # bugs), but doing this while maintaining backward compatibility # with the previous, context-unaware version get_memmaping_executor # exposes too many low-level details. context_id = uuid4().hex self.set_current_context(context_id) def set_current_context(self, context_id): self._current_context_id = context_id self.register_new_context(context_id) def register_new_context(self, context_id): # Prepare a sub-folder name specific to a context (usually a unique id # generated by each instance of the Parallel class). Do not create in # advance to spare FS write access if no array is to be dumped). if context_id in self._cached_temp_folders: return else: # During its lifecycle, one Parallel object can have several # executors associated to it (for instance, if a loky worker raises # an exception, joblib shutdowns the executor and instantly # recreates a new one before raising the error - see # ``ensure_ready``. Because we don't want two executors tied to # the same Parallel object (and thus the same context id) to # register/use/delete the same folder, we also add an id specific # to the current Manager (and thus specific to its associated # executor) to the folder name. new_folder_name = ( "joblib_memmapping_folder_{}_{}_{}".format( os.getpid(), self._id, context_id) ) new_folder_path, _ = _get_temp_dir( new_folder_name, self._temp_folder_root ) self.register_folder_finalizer(new_folder_path, context_id) self._cached_temp_folders[context_id] = new_folder_path def resolve_temp_folder_name(self): """Return a folder name specific to the currently activated context""" return self._cached_temp_folders[self._current_context_id] # resource management API def register_folder_finalizer(self, pool_subfolder, context_id): # Register the garbage collector at program exit in case caller forgets # to call terminate explicitly: note we do not pass any reference to # ensure that this callback won't prevent garbage collection of # parallel instance and related file handler resources such as POSIX # semaphores and pipes pool_module_name = whichmodule(delete_folder, 'delete_folder') resource_tracker.register(pool_subfolder, "folder") def _cleanup(): # In some cases the Python runtime seems to set delete_folder to # None just before exiting when accessing the delete_folder # function from the closure namespace. So instead we reimport # the delete_folder function explicitly. # https://github.com/joblib/joblib/issues/328 # We cannot just use from 'joblib.pool import delete_folder' # because joblib should only use relative imports to allow # easy vendoring. delete_folder = __import__( pool_module_name, fromlist=['delete_folder'] ).delete_folder try: delete_folder(pool_subfolder, allow_non_empty=True) resource_tracker.unregister(pool_subfolder, "folder") except OSError: warnings.warn("Failed to delete temporary folder: {}" .format(pool_subfolder)) self._finalizers[context_id] = atexit.register(_cleanup) def _clean_temporary_resources(self, context_id=None, force=False, allow_non_empty=False): """Clean temporary resources created by a process-based pool""" if context_id is None: # Iterates over a copy of the cache keys to avoid Error due to # iterating over a changing size dictionary. for context_id in list(self._cached_temp_folders): self._clean_temporary_resources( context_id, force=force, allow_non_empty=allow_non_empty ) else: temp_folder = self._cached_temp_folders.get(context_id) if temp_folder and os.path.exists(temp_folder): for filename in os.listdir(temp_folder): if force: # Some workers have failed and the ref counted might # be off. The workers should have shut down by this # time so forcefully clean up the files. resource_tracker.unregister( os.path.join(temp_folder, filename), "file" ) else: resource_tracker.maybe_unlink( os.path.join(temp_folder, filename), "file" ) # When forcing clean-up, try to delete the folder even if some # files are still in it. Otherwise, try to delete the folder allow_non_empty |= force # Clean up the folder if possible, either if it is empty or # if none of the files in it are in used and allow_non_empty. try: delete_folder( temp_folder, allow_non_empty=allow_non_empty ) # Forget the folder once it has been deleted self._cached_temp_folders.pop(context_id, None) resource_tracker.unregister(temp_folder, "folder") # Also cancel the finalizers that gets triggered at gc. finalizer = self._finalizers.pop(context_id, None) if finalizer is not None: atexit.unregister(finalizer) except OSError: # Temporary folder cannot be deleted right now. # This folder will be cleaned up by an atexit # finalizer registered by the memmapping_reducer. pass joblib-1.3.2/joblib/_multiprocessing_helpers.py000066400000000000000000000036051446465525000217200ustar00rootroot00000000000000"""Helper module to factorize the conditional multiprocessing import logic We use a distinct module to simplify import statements and avoid introducing circular dependencies (for instance for the assert_spawning name). """ import os import warnings # Obtain possible configuration from the environment, assuming 1 (on) # by default, upon 0 set to None. Should instructively fail if some non # 0/1 value is set. mp = int(os.environ.get('JOBLIB_MULTIPROCESSING', 1)) or None if mp: try: import multiprocessing as mp import _multiprocessing # noqa except ImportError: mp = None # 2nd stage: validate that locking is available on the system and # issue a warning if not if mp is not None: try: # try to create a named semaphore using SemLock to make sure they are # available on this platform. We use the low level object # _multiprocessing.SemLock to avoid spawning a resource tracker on # Unix system or changing the default backend. import tempfile from _multiprocessing import SemLock _rand = tempfile._RandomNameSequence() for i in range(100): try: name = '/joblib-{}-{}' .format( os.getpid(), next(_rand)) _sem = SemLock(0, 0, 1, name=name, unlink=True) del _sem # cleanup break except FileExistsError as e: # pragma: no cover if i >= 99: raise FileExistsError( 'cannot find name for semaphore') from e except (FileExistsError, AttributeError, ImportError, OSError) as e: mp = None warnings.warn('%s. joblib will operate in serial mode' % (e,)) # 3rd stage: backward compat for the assert_spawning helper if mp is not None: from multiprocessing.context import assert_spawning else: assert_spawning = None joblib-1.3.2/joblib/_parallel_backends.py000066400000000000000000000625031446465525000203770ustar00rootroot00000000000000""" Backends for embarrassingly parallel code. """ import gc import os import warnings import threading import contextlib from abc import ABCMeta, abstractmethod from ._multiprocessing_helpers import mp if mp is not None: from .pool import MemmappingPool from multiprocessing.pool import ThreadPool from .executor import get_memmapping_executor # Import loky only if multiprocessing is present from .externals.loky import process_executor, cpu_count from .externals.loky.process_executor import ShutdownExecutorError from .externals.loky.process_executor import _ExceptionWithTraceback class ParallelBackendBase(metaclass=ABCMeta): """Helper abc which defines all methods a ParallelBackend must implement""" supports_inner_max_num_threads = False supports_retrieve_callback = False default_n_jobs = 1 @property def supports_return_generator(self): return self.supports_retrieve_callback @property def supports_timeout(self): return self.supports_retrieve_callback nesting_level = None def __init__(self, nesting_level=None, inner_max_num_threads=None, **kwargs): super().__init__(**kwargs) self.nesting_level = nesting_level self.inner_max_num_threads = inner_max_num_threads MAX_NUM_THREADS_VARS = [ 'OMP_NUM_THREADS', 'OPENBLAS_NUM_THREADS', 'MKL_NUM_THREADS', 'BLIS_NUM_THREADS', 'VECLIB_MAXIMUM_THREADS', 'NUMBA_NUM_THREADS', 'NUMEXPR_NUM_THREADS', ] TBB_ENABLE_IPC_VAR = "ENABLE_IPC" @abstractmethod def effective_n_jobs(self, n_jobs): """Determine the number of jobs that can actually run in parallel n_jobs is the number of workers requested by the callers. Passing n_jobs=-1 means requesting all available workers for instance matching the number of CPU cores on the worker host(s). This method should return a guesstimate of the number of workers that can actually perform work concurrently. The primary use case is to make it possible for the caller to know in how many chunks to slice the work. In general working on larger data chunks is more efficient (less scheduling overhead and better use of CPU cache prefetching heuristics) as long as all the workers have enough work to do. """ @abstractmethod def apply_async(self, func, callback=None): """Schedule a func to be run""" def retrieve_result_callback(self, out): """Called within the callback function passed in apply_async. The argument of this function is the argument given to a callback in the considered backend. It is supposed to return the outcome of a task if it succeeded or raise the exception if it failed. """ def configure(self, n_jobs=1, parallel=None, prefer=None, require=None, **backend_args): """Reconfigure the backend and return the number of workers. This makes it possible to reuse an existing backend instance for successive independent calls to Parallel with different parameters. """ self.parallel = parallel return self.effective_n_jobs(n_jobs) def start_call(self): """Call-back method called at the beginning of a Parallel call""" def stop_call(self): """Call-back method called at the end of a Parallel call""" def terminate(self): """Shutdown the workers and free the shared memory.""" def compute_batch_size(self): """Determine the optimal batch size""" return 1 def batch_completed(self, batch_size, duration): """Callback indicate how long it took to run a batch""" def get_exceptions(self): """List of exception types to be captured.""" return [] def abort_everything(self, ensure_ready=True): """Abort any running tasks This is called when an exception has been raised when executing a task and all the remaining tasks will be ignored and can therefore be aborted to spare computation resources. If ensure_ready is True, the backend should be left in an operating state as future tasks might be re-submitted via that same backend instance. If ensure_ready is False, the implementer of this method can decide to leave the backend in a closed / terminated state as no new task are expected to be submitted to this backend. Setting ensure_ready to False is an optimization that can be leveraged when aborting tasks via killing processes from a local process pool managed by the backend it-self: if we expect no new tasks, there is no point in re-creating new workers. """ # Does nothing by default: to be overridden in subclasses when # canceling tasks is possible. pass def get_nested_backend(self): """Backend instance to be used by nested Parallel calls. By default a thread-based backend is used for the first level of nesting. Beyond, switch to sequential backend to avoid spawning too many threads on the host. """ nesting_level = getattr(self, 'nesting_level', 0) + 1 if nesting_level > 1: return SequentialBackend(nesting_level=nesting_level), None else: return ThreadingBackend(nesting_level=nesting_level), None @contextlib.contextmanager def retrieval_context(self): """Context manager to manage an execution context. Calls to Parallel.retrieve will be made inside this context. By default, this does nothing. It may be useful for subclasses to handle nested parallelism. In particular, it may be required to avoid deadlocks if a backend manages a fixed number of workers, when those workers may be asked to do nested Parallel calls. Without 'retrieval_context' this could lead to deadlock, as all the workers managed by the backend may be "busy" waiting for the nested parallel calls to finish, but the backend has no free workers to execute those tasks. """ yield def _prepare_worker_env(self, n_jobs): """Return environment variables limiting threadpools in external libs. This function return a dict containing environment variables to pass when creating a pool of process. These environment variables limit the number of threads to `n_threads` for OpenMP, MKL, Accelerated and OpenBLAS libraries in the child processes. """ explicit_n_threads = self.inner_max_num_threads default_n_threads = str(max(cpu_count() // n_jobs, 1)) # Set the inner environment variables to self.inner_max_num_threads if # it is given. Else, default to cpu_count // n_jobs unless the variable # is already present in the parent process environment. env = {} for var in self.MAX_NUM_THREADS_VARS: if explicit_n_threads is None: var_value = os.environ.get(var, None) if var_value is None: var_value = default_n_threads else: var_value = str(explicit_n_threads) env[var] = var_value if self.TBB_ENABLE_IPC_VAR not in os.environ: # To avoid over-subscription when using TBB, let the TBB schedulers # use Inter Process Communication to coordinate: env[self.TBB_ENABLE_IPC_VAR] = "1" return env @staticmethod def in_main_thread(): return isinstance(threading.current_thread(), threading._MainThread) class SequentialBackend(ParallelBackendBase): """A ParallelBackend which will execute all batches sequentially. Does not use/create any threading objects, and hence has minimal overhead. Used when n_jobs == 1. """ uses_threads = True supports_timeout = False supports_retrieve_callback = False supports_sharedmem = True def effective_n_jobs(self, n_jobs): """Determine the number of jobs which are going to run in parallel""" if n_jobs == 0: raise ValueError('n_jobs == 0 in Parallel has no meaning') return 1 def apply_async(self, func, callback=None): """Schedule a func to be run""" raise RuntimeError("Should never be called for SequentialBackend.") def retrieve_result_callback(self, out): raise RuntimeError("Should never be called for SequentialBackend.") def get_nested_backend(self): # import is not top level to avoid cyclic import errors. from .parallel import get_active_backend # SequentialBackend should neither change the nesting level, the # default backend or the number of jobs. Just return the current one. return get_active_backend() class PoolManagerMixin(object): """A helper class for managing pool of workers.""" _pool = None def effective_n_jobs(self, n_jobs): """Determine the number of jobs which are going to run in parallel""" if n_jobs == 0: raise ValueError('n_jobs == 0 in Parallel has no meaning') elif mp is None or n_jobs is None: # multiprocessing is not available or disabled, fallback # to sequential mode return 1 elif n_jobs < 0: n_jobs = max(cpu_count() + 1 + n_jobs, 1) return n_jobs def terminate(self): """Shutdown the process or thread pool""" if self._pool is not None: self._pool.close() self._pool.terminate() # terminate does a join() self._pool = None def _get_pool(self): """Used by apply_async to make it possible to implement lazy init""" return self._pool @staticmethod def _wrap_func_call(func): """Protect function call and return error with traceback.""" try: return func() except BaseException as e: return _ExceptionWithTraceback(e) def apply_async(self, func, callback=None): """Schedule a func to be run""" # Here, we need a wrapper to avoid crashes on KeyboardInterruptErrors. # We also call the callback on error, to make sure the pool does not # wait on crashed jobs. return self._get_pool().apply_async( self._wrap_func_call, (func,), callback=callback, error_callback=callback ) def retrieve_result_callback(self, out): """Mimic concurrent.futures results, raising an error if needed.""" if isinstance(out, _ExceptionWithTraceback): rebuild, args = out.__reduce__() out = rebuild(*args) if isinstance(out, BaseException): raise out return out def abort_everything(self, ensure_ready=True): """Shutdown the pool and restart a new one with the same parameters""" self.terminate() if ensure_ready: self.configure(n_jobs=self.parallel.n_jobs, parallel=self.parallel, **self.parallel._backend_args) class AutoBatchingMixin(object): """A helper class for automagically batching jobs.""" # In seconds, should be big enough to hide multiprocessing dispatching # overhead. # This settings was found by running benchmarks/bench_auto_batching.py # with various parameters on various platforms. MIN_IDEAL_BATCH_DURATION = .2 # Should not be too high to avoid stragglers: long jobs running alone # on a single worker while other workers have no work to process any more. MAX_IDEAL_BATCH_DURATION = 2 # Batching counters default values _DEFAULT_EFFECTIVE_BATCH_SIZE = 1 _DEFAULT_SMOOTHED_BATCH_DURATION = 0.0 def __init__(self, **kwargs): super().__init__(**kwargs) self._effective_batch_size = self._DEFAULT_EFFECTIVE_BATCH_SIZE self._smoothed_batch_duration = self._DEFAULT_SMOOTHED_BATCH_DURATION def compute_batch_size(self): """Determine the optimal batch size""" old_batch_size = self._effective_batch_size batch_duration = self._smoothed_batch_duration if (batch_duration > 0 and batch_duration < self.MIN_IDEAL_BATCH_DURATION): # The current batch size is too small: the duration of the # processing of a batch of task is not large enough to hide # the scheduling overhead. ideal_batch_size = int(old_batch_size * self.MIN_IDEAL_BATCH_DURATION / batch_duration) # Multiply by two to limit oscilations between min and max. ideal_batch_size *= 2 # dont increase the batch size too fast to limit huge batch sizes # potentially leading to starving worker batch_size = min(2 * old_batch_size, ideal_batch_size) batch_size = max(batch_size, 1) self._effective_batch_size = batch_size if self.parallel.verbose >= 10: self.parallel._print( f"Batch computation too fast ({batch_duration}s.) " f"Setting batch_size={batch_size}." ) elif (batch_duration > self.MAX_IDEAL_BATCH_DURATION and old_batch_size >= 2): # The current batch size is too big. If we schedule overly long # running batches some CPUs might wait with nothing left to do # while a couple of CPUs a left processing a few long running # batches. Better reduce the batch size a bit to limit the # likelihood of scheduling such stragglers. # decrease the batch size quickly to limit potential starving ideal_batch_size = int( old_batch_size * self.MIN_IDEAL_BATCH_DURATION / batch_duration ) # Multiply by two to limit oscilations between min and max. batch_size = max(2 * ideal_batch_size, 1) self._effective_batch_size = batch_size if self.parallel.verbose >= 10: self.parallel._print( f"Batch computation too slow ({batch_duration}s.) " f"Setting batch_size={batch_size}." ) else: # No batch size adjustment batch_size = old_batch_size if batch_size != old_batch_size: # Reset estimation of the smoothed mean batch duration: this # estimate is updated in the multiprocessing apply_async # CallBack as long as the batch_size is constant. Therefore # we need to reset the estimate whenever we re-tune the batch # size. self._smoothed_batch_duration = \ self._DEFAULT_SMOOTHED_BATCH_DURATION return batch_size def batch_completed(self, batch_size, duration): """Callback indicate how long it took to run a batch""" if batch_size == self._effective_batch_size: # Update the smoothed streaming estimate of the duration of a batch # from dispatch to completion old_duration = self._smoothed_batch_duration if old_duration == self._DEFAULT_SMOOTHED_BATCH_DURATION: # First record of duration for this batch size after the last # reset. new_duration = duration else: # Update the exponentially weighted average of the duration of # batch for the current effective size. new_duration = 0.8 * old_duration + 0.2 * duration self._smoothed_batch_duration = new_duration def reset_batch_stats(self): """Reset batch statistics to default values. This avoids interferences with future jobs. """ self._effective_batch_size = self._DEFAULT_EFFECTIVE_BATCH_SIZE self._smoothed_batch_duration = self._DEFAULT_SMOOTHED_BATCH_DURATION class ThreadingBackend(PoolManagerMixin, ParallelBackendBase): """A ParallelBackend which will use a thread pool to execute batches in. This is a low-overhead backend but it suffers from the Python Global Interpreter Lock if the called function relies a lot on Python objects. Mostly useful when the execution bottleneck is a compiled extension that explicitly releases the GIL (for instance a Cython loop wrapped in a "with nogil" block or an expensive call to a library such as NumPy). The actual thread pool is lazily initialized: the actual thread pool construction is delayed to the first call to apply_async. ThreadingBackend is used as the default backend for nested calls. """ supports_retrieve_callback = True uses_threads = True supports_sharedmem = True def configure(self, n_jobs=1, parallel=None, **backend_args): """Build a process or thread pool and return the number of workers""" n_jobs = self.effective_n_jobs(n_jobs) if n_jobs == 1: # Avoid unnecessary overhead and use sequential backend instead. raise FallbackToBackend( SequentialBackend(nesting_level=self.nesting_level)) self.parallel = parallel self._n_jobs = n_jobs return n_jobs def _get_pool(self): """Lazily initialize the thread pool The actual pool of worker threads is only initialized at the first call to apply_async. """ if self._pool is None: self._pool = ThreadPool(self._n_jobs) return self._pool class MultiprocessingBackend(PoolManagerMixin, AutoBatchingMixin, ParallelBackendBase): """A ParallelBackend which will use a multiprocessing.Pool. Will introduce some communication and memory overhead when exchanging input and output data with the with the worker Python processes. However, does not suffer from the Python Global Interpreter Lock. """ supports_retrieve_callback = True supports_return_generator = False def effective_n_jobs(self, n_jobs): """Determine the number of jobs which are going to run in parallel. This also checks if we are attempting to create a nested parallel loop. """ if mp is None: return 1 if mp.current_process().daemon: # Daemonic processes cannot have children if n_jobs != 1: if inside_dask_worker(): msg = ( "Inside a Dask worker with daemon=True, " "setting n_jobs=1.\nPossible work-arounds:\n" "- dask.config.set(" "{'distributed.worker.daemon': False})" "- set the environment variable " "DASK_DISTRIBUTED__WORKER__DAEMON=False\n" "before creating your Dask cluster." ) else: msg = ( 'Multiprocessing-backed parallel loops ' 'cannot be nested, setting n_jobs=1' ) warnings.warn(msg, stacklevel=3) return 1 if process_executor._CURRENT_DEPTH > 0: # Mixing loky and multiprocessing in nested loop is not supported if n_jobs != 1: warnings.warn( 'Multiprocessing-backed parallel loops cannot be nested,' ' below loky, setting n_jobs=1', stacklevel=3) return 1 elif not (self.in_main_thread() or self.nesting_level == 0): # Prevent posix fork inside in non-main posix threads if n_jobs != 1: warnings.warn( 'Multiprocessing-backed parallel loops cannot be nested' ' below threads, setting n_jobs=1', stacklevel=3) return 1 return super(MultiprocessingBackend, self).effective_n_jobs(n_jobs) def configure(self, n_jobs=1, parallel=None, prefer=None, require=None, **memmappingpool_args): """Build a process or thread pool and return the number of workers""" n_jobs = self.effective_n_jobs(n_jobs) if n_jobs == 1: raise FallbackToBackend( SequentialBackend(nesting_level=self.nesting_level)) # Make sure to free as much memory as possible before forking gc.collect() self._pool = MemmappingPool(n_jobs, **memmappingpool_args) self.parallel = parallel return n_jobs def terminate(self): """Shutdown the process or thread pool""" super(MultiprocessingBackend, self).terminate() self.reset_batch_stats() class LokyBackend(AutoBatchingMixin, ParallelBackendBase): """Managing pool of workers with loky instead of multiprocessing.""" supports_retrieve_callback = True supports_inner_max_num_threads = True def configure(self, n_jobs=1, parallel=None, prefer=None, require=None, idle_worker_timeout=300, **memmappingexecutor_args): """Build a process executor and return the number of workers""" n_jobs = self.effective_n_jobs(n_jobs) if n_jobs == 1: raise FallbackToBackend( SequentialBackend(nesting_level=self.nesting_level)) self._workers = get_memmapping_executor( n_jobs, timeout=idle_worker_timeout, env=self._prepare_worker_env(n_jobs=n_jobs), context_id=parallel._id, **memmappingexecutor_args) self.parallel = parallel return n_jobs def effective_n_jobs(self, n_jobs): """Determine the number of jobs which are going to run in parallel""" if n_jobs == 0: raise ValueError('n_jobs == 0 in Parallel has no meaning') elif mp is None or n_jobs is None: # multiprocessing is not available or disabled, fallback # to sequential mode return 1 elif mp.current_process().daemon: # Daemonic processes cannot have children if n_jobs != 1: if inside_dask_worker(): msg = ( "Inside a Dask worker with daemon=True, " "setting n_jobs=1.\nPossible work-arounds:\n" "- dask.config.set(" "{'distributed.worker.daemon': False})\n" "- set the environment variable " "DASK_DISTRIBUTED__WORKER__DAEMON=False\n" "before creating your Dask cluster." ) else: msg = ( 'Loky-backed parallel loops cannot be called in a' ' multiprocessing, setting n_jobs=1' ) warnings.warn(msg, stacklevel=3) return 1 elif not (self.in_main_thread() or self.nesting_level == 0): # Prevent posix fork inside in non-main posix threads if n_jobs != 1: warnings.warn( 'Loky-backed parallel loops cannot be nested below ' 'threads, setting n_jobs=1', stacklevel=3) return 1 elif n_jobs < 0: n_jobs = max(cpu_count() + 1 + n_jobs, 1) return n_jobs def apply_async(self, func, callback=None): """Schedule a func to be run""" future = self._workers.submit(func) if callback is not None: future.add_done_callback(callback) return future def retrieve_result_callback(self, out): try: return out.result() except ShutdownExecutorError: raise RuntimeError( "The executor underlying Parallel has been shutdown. " "This is likely due to the garbage collection of a previous " "generator from a call to Parallel with return_as='generator'." " Make sure the generator is not garbage collected when " "submitting a new job or that it is first properly exhausted." ) def terminate(self): if self._workers is not None: # Don't terminate the workers as we want to reuse them in later # calls, but cleanup the temporary resources that the Parallel call # created. This 'hack' requires a private, low-level operation. self._workers._temp_folder_manager._clean_temporary_resources( context_id=self.parallel._id, force=False ) self._workers = None self.reset_batch_stats() def abort_everything(self, ensure_ready=True): """Shutdown the workers and restart a new one with the same parameters """ self._workers.terminate(kill_workers=True) self._workers = None if ensure_ready: self.configure(n_jobs=self.parallel.n_jobs, parallel=self.parallel) class FallbackToBackend(Exception): """Raised when configuration should fallback to another backend""" def __init__(self, backend): self.backend = backend def inside_dask_worker(): """Check whether the current function is executed inside a Dask worker. """ # This function can not be in joblib._dask because there would be a # circular import: # _dask imports _parallel_backend that imports _dask ... try: from distributed import get_worker except ImportError: return False try: get_worker() return True except ValueError: return False joblib-1.3.2/joblib/_store_backends.py000066400000000000000000000373441446465525000177440ustar00rootroot00000000000000"""Storage providers backends for Memory caching.""" from pickle import PicklingError import re import os import os.path import datetime import json import shutil import warnings import collections import operator import threading from abc import ABCMeta, abstractmethod from .backports import concurrency_safe_rename from .disk import mkdirp, memstr_to_bytes, rm_subdirs from . import numpy_pickle CacheItemInfo = collections.namedtuple('CacheItemInfo', 'path size last_access') class CacheWarning(Warning): """Warning to capture dump failures except for PicklingError.""" pass def concurrency_safe_write(object_to_write, filename, write_func): """Writes an object into a unique file in a concurrency-safe way.""" thread_id = id(threading.current_thread()) temporary_filename = '{}.thread-{}-pid-{}'.format( filename, thread_id, os.getpid()) write_func(object_to_write, temporary_filename) return temporary_filename class StoreBackendBase(metaclass=ABCMeta): """Helper Abstract Base Class which defines all methods that a StorageBackend must implement.""" location = None @abstractmethod def _open_item(self, f, mode): """Opens an item on the store and return a file-like object. This method is private and only used by the StoreBackendMixin object. Parameters ---------- f: a file-like object The file-like object where an item is stored and retrieved mode: string, optional the mode in which the file-like object is opened allowed valued are 'rb', 'wb' Returns ------- a file-like object """ @abstractmethod def _item_exists(self, location): """Checks if an item location exists in the store. This method is private and only used by the StoreBackendMixin object. Parameters ---------- location: string The location of an item. On a filesystem, this corresponds to the absolute path, including the filename, of a file. Returns ------- True if the item exists, False otherwise """ @abstractmethod def _move_item(self, src, dst): """Moves an item from src to dst in the store. This method is private and only used by the StoreBackendMixin object. Parameters ---------- src: string The source location of an item dst: string The destination location of an item """ @abstractmethod def create_location(self, location): """Creates a location on the store. Parameters ---------- location: string The location in the store. On a filesystem, this corresponds to a directory. """ @abstractmethod def clear_location(self, location): """Clears a location on the store. Parameters ---------- location: string The location in the store. On a filesystem, this corresponds to a directory or a filename absolute path """ @abstractmethod def get_items(self): """Returns the whole list of items available in the store. Returns ------- The list of items identified by their ids (e.g filename in a filesystem). """ @abstractmethod def configure(self, location, verbose=0, backend_options=dict()): """Configures the store. Parameters ---------- location: string The base location used by the store. On a filesystem, this corresponds to a directory. verbose: int The level of verbosity of the store backend_options: dict Contains a dictionary of named parameters used to configure the store backend. """ class StoreBackendMixin(object): """Class providing all logic for managing the store in a generic way. The StoreBackend subclass has to implement 3 methods: create_location, clear_location and configure. The StoreBackend also has to provide a private _open_item, _item_exists and _move_item methods. The _open_item method has to have the same signature as the builtin open and return a file-like object. """ def load_item(self, path, verbose=1, msg=None): """Load an item from the store given its path as a list of strings.""" full_path = os.path.join(self.location, *path) if verbose > 1: if verbose < 10: print('{0}...'.format(msg)) else: print('{0} from {1}'.format(msg, full_path)) mmap_mode = (None if not hasattr(self, 'mmap_mode') else self.mmap_mode) filename = os.path.join(full_path, 'output.pkl') if not self._item_exists(filename): raise KeyError("Non-existing item (may have been " "cleared).\nFile %s does not exist" % filename) # file-like object cannot be used when mmap_mode is set if mmap_mode is None: with self._open_item(filename, "rb") as f: item = numpy_pickle.load(f) else: item = numpy_pickle.load(filename, mmap_mode=mmap_mode) return item def dump_item(self, path, item, verbose=1): """Dump an item in the store at the path given as a list of strings.""" try: item_path = os.path.join(self.location, *path) if not self._item_exists(item_path): self.create_location(item_path) filename = os.path.join(item_path, 'output.pkl') if verbose > 10: print('Persisting in %s' % item_path) def write_func(to_write, dest_filename): with self._open_item(dest_filename, "wb") as f: try: numpy_pickle.dump(to_write, f, compress=self.compress) except PicklingError as e: # TODO(1.5) turn into error warnings.warn( "Unable to cache to disk: failed to pickle " "output. In version 1.5 this will raise an " f"exception. Exception: {e}.", FutureWarning ) self._concurrency_safe_write(item, filename, write_func) except Exception as e: # noqa: E722 warnings.warn( "Unable to cache to disk. Possibly a race condition in the " f"creation of the directory. Exception: {e}.", CacheWarning ) def clear_item(self, path): """Clear the item at the path, given as a list of strings.""" item_path = os.path.join(self.location, *path) if self._item_exists(item_path): self.clear_location(item_path) def contains_item(self, path): """Check if there is an item at the path, given as a list of strings""" item_path = os.path.join(self.location, *path) filename = os.path.join(item_path, 'output.pkl') return self._item_exists(filename) def get_item_info(self, path): """Return information about item.""" return {'location': os.path.join(self.location, *path)} def get_metadata(self, path): """Return actual metadata of an item.""" try: item_path = os.path.join(self.location, *path) filename = os.path.join(item_path, 'metadata.json') with self._open_item(filename, 'rb') as f: return json.loads(f.read().decode('utf-8')) except: # noqa: E722 return {} def store_metadata(self, path, metadata): """Store metadata of a computation.""" try: item_path = os.path.join(self.location, *path) self.create_location(item_path) filename = os.path.join(item_path, 'metadata.json') def write_func(to_write, dest_filename): with self._open_item(dest_filename, "wb") as f: f.write(json.dumps(to_write).encode('utf-8')) self._concurrency_safe_write(metadata, filename, write_func) except: # noqa: E722 pass def contains_path(self, path): """Check cached function is available in store.""" func_path = os.path.join(self.location, *path) return self.object_exists(func_path) def clear_path(self, path): """Clear all items with a common path in the store.""" func_path = os.path.join(self.location, *path) if self._item_exists(func_path): self.clear_location(func_path) def store_cached_func_code(self, path, func_code=None): """Store the code of the cached function.""" func_path = os.path.join(self.location, *path) if not self._item_exists(func_path): self.create_location(func_path) if func_code is not None: filename = os.path.join(func_path, "func_code.py") with self._open_item(filename, 'wb') as f: f.write(func_code.encode('utf-8')) def get_cached_func_code(self, path): """Store the code of the cached function.""" path += ['func_code.py', ] filename = os.path.join(self.location, *path) try: with self._open_item(filename, 'rb') as f: return f.read().decode('utf-8') except: # noqa: E722 raise def get_cached_func_info(self, path): """Return information related to the cached function if it exists.""" return {'location': os.path.join(self.location, *path)} def clear(self): """Clear the whole store content.""" self.clear_location(self.location) def enforce_store_limits( self, bytes_limit, items_limit=None, age_limit=None ): """ Remove the store's oldest files to enforce item, byte, and age limits. """ items_to_delete = self._get_items_to_delete( bytes_limit, items_limit, age_limit ) for item in items_to_delete: if self.verbose > 10: print('Deleting item {0}'.format(item)) try: self.clear_location(item.path) except OSError: # Even with ignore_errors=True shutil.rmtree can raise OSError # with: # [Errno 116] Stale file handle if another process has deleted # the folder already. pass def _get_items_to_delete( self, bytes_limit, items_limit=None, age_limit=None ): """ Get items to delete to keep the store under size, file, & age limits. """ if isinstance(bytes_limit, str): bytes_limit = memstr_to_bytes(bytes_limit) items = self.get_items() size = sum(item.size for item in items) if bytes_limit is not None: to_delete_size = size - bytes_limit else: to_delete_size = 0 if items_limit is not None: to_delete_items = len(items) - items_limit else: to_delete_items = 0 if age_limit is not None: older_item = min(item.last_access for item in items) deadline = datetime.datetime.now() - age_limit else: deadline = None if ( to_delete_size <= 0 and to_delete_items <= 0 and (deadline is None or older_item > deadline) ): return [] # We want to delete first the cache items that were accessed a # long time ago items.sort(key=operator.attrgetter('last_access')) items_to_delete = [] size_so_far = 0 items_so_far = 0 for item in items: if ( (size_so_far >= to_delete_size) and items_so_far >= to_delete_items and (deadline is None or deadline < item.last_access) ): break items_to_delete.append(item) size_so_far += item.size items_so_far += 1 return items_to_delete def _concurrency_safe_write(self, to_write, filename, write_func): """Writes an object into a file in a concurrency-safe way.""" temporary_filename = concurrency_safe_write(to_write, filename, write_func) self._move_item(temporary_filename, filename) def __repr__(self): """Printable representation of the store location.""" return '{class_name}(location="{location}")'.format( class_name=self.__class__.__name__, location=self.location) class FileSystemStoreBackend(StoreBackendBase, StoreBackendMixin): """A StoreBackend used with local or network file systems.""" _open_item = staticmethod(open) _item_exists = staticmethod(os.path.exists) _move_item = staticmethod(concurrency_safe_rename) def clear_location(self, location): """Delete location on store.""" if (location == self.location): rm_subdirs(location) else: shutil.rmtree(location, ignore_errors=True) def create_location(self, location): """Create object location on store""" mkdirp(location) def get_items(self): """Returns the whole list of items available in the store.""" items = [] for dirpath, _, filenames in os.walk(self.location): is_cache_hash_dir = re.match('[a-f0-9]{32}', os.path.basename(dirpath)) if is_cache_hash_dir: output_filename = os.path.join(dirpath, 'output.pkl') try: last_access = os.path.getatime(output_filename) except OSError: try: last_access = os.path.getatime(dirpath) except OSError: # The directory has already been deleted continue last_access = datetime.datetime.fromtimestamp(last_access) try: full_filenames = [os.path.join(dirpath, fn) for fn in filenames] dirsize = sum(os.path.getsize(fn) for fn in full_filenames) except OSError: # Either output_filename or one of the files in # dirpath does not exist any more. We assume this # directory is being cleaned by another process already continue items.append(CacheItemInfo(dirpath, dirsize, last_access)) return items def configure(self, location, verbose=1, backend_options=None): """Configure the store backend. For this backend, valid store options are 'compress' and 'mmap_mode' """ if backend_options is None: backend_options = {} # setup location directory self.location = location if not os.path.exists(self.location): mkdirp(self.location) # item can be stored compressed for faster I/O self.compress = backend_options.get('compress', False) # FileSystemStoreBackend can be used with mmap_mode options under # certain conditions. mmap_mode = backend_options.get('mmap_mode') if self.compress and mmap_mode is not None: warnings.warn('Compressed items cannot be memmapped in a ' 'filesystem store. Option will be ignored.', stacklevel=2) self.mmap_mode = mmap_mode self.verbose = verbose joblib-1.3.2/joblib/_utils.py000066400000000000000000000025151446465525000161060ustar00rootroot00000000000000# Adapted from https://stackoverflow.com/a/9558001/2536294 import ast from dataclasses import dataclass import operator as op # supported operators operators = { ast.Add: op.add, ast.Sub: op.sub, ast.Mult: op.mul, ast.Div: op.truediv, ast.FloorDiv: op.floordiv, ast.Mod: op.mod, ast.Pow: op.pow, ast.USub: op.neg, } def eval_expr(expr): """ >>> eval_expr('2*6') 12 >>> eval_expr('2**6') 64 >>> eval_expr('1 + 2*3**(4) / (6 + -7)') -161.0 """ try: return eval_(ast.parse(expr, mode="eval").body) except (TypeError, SyntaxError, KeyError) as e: raise ValueError( f"{expr!r} is not a valid or supported arithmetic expression." ) from e def eval_(node): if isinstance(node, ast.Num): # return node.n elif isinstance(node, ast.BinOp): # return operators[type(node.op)](eval_(node.left), eval_(node.right)) elif isinstance(node, ast.UnaryOp): # e.g., -1 return operators[type(node.op)](eval_(node.operand)) else: raise TypeError(node) @dataclass(frozen=True) class _Sentinel: """A sentinel to mark a parameter as not explicitly set""" default_value: object def __repr__(self): return f"default({self.default_value!r})" joblib-1.3.2/joblib/backports.py000066400000000000000000000123611446465525000165770ustar00rootroot00000000000000""" Backports of fixes for joblib dependencies """ import os import re import time from os.path import basename from multiprocessing import util class Version: """Backport from deprecated distutils We maintain this backport to avoid introducing a new dependency on `packaging`. We might rexplore this choice in the future if all major Python projects introduce a dependency on packaging anyway. """ def __init__(self, vstring=None): if vstring: self.parse(vstring) def __repr__(self): return "%s ('%s')" % (self.__class__.__name__, str(self)) def __eq__(self, other): c = self._cmp(other) if c is NotImplemented: return c return c == 0 def __lt__(self, other): c = self._cmp(other) if c is NotImplemented: return c return c < 0 def __le__(self, other): c = self._cmp(other) if c is NotImplemented: return c return c <= 0 def __gt__(self, other): c = self._cmp(other) if c is NotImplemented: return c return c > 0 def __ge__(self, other): c = self._cmp(other) if c is NotImplemented: return c return c >= 0 class LooseVersion(Version): """Backport from deprecated distutils We maintain this backport to avoid introducing a new dependency on `packaging`. We might rexplore this choice in the future if all major Python projects introduce a dependency on packaging anyway. """ component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) def __init__(self, vstring=None): if vstring: self.parse(vstring) def parse(self, vstring): # I've given up on thinking I can reconstruct the version string # from the parsed tuple -- so I just store the string here for # use by __str__ self.vstring = vstring components = [x for x in self.component_re.split(vstring) if x and x != '.'] for i, obj in enumerate(components): try: components[i] = int(obj) except ValueError: pass self.version = components def __str__(self): return self.vstring def __repr__(self): return "LooseVersion ('%s')" % str(self) def _cmp(self, other): if isinstance(other, str): other = LooseVersion(other) elif not isinstance(other, LooseVersion): return NotImplemented if self.version == other.version: return 0 if self.version < other.version: return -1 if self.version > other.version: return 1 try: import numpy as np def make_memmap(filename, dtype='uint8', mode='r+', offset=0, shape=None, order='C', unlink_on_gc_collect=False): """Custom memmap constructor compatible with numpy.memmap. This function: - is a backport the numpy memmap offset fix (See https://github.com/numpy/numpy/pull/8443 for more details. The numpy fix is available starting numpy 1.13) - adds ``unlink_on_gc_collect``, which specifies explicitly whether the process re-constructing the memmap owns a reference to the underlying file. If set to True, it adds a finalizer to the newly-created memmap that sends a maybe_unlink request for the memmaped file to resource_tracker. """ util.debug( "[MEMMAP READ] creating a memmap (shape {}, filename {}, " "pid {})".format(shape, basename(filename), os.getpid()) ) mm = np.memmap(filename, dtype=dtype, mode=mode, offset=offset, shape=shape, order=order) if LooseVersion(np.__version__) < '1.13': mm.offset = offset if unlink_on_gc_collect: from ._memmapping_reducer import add_maybe_unlink_finalizer add_maybe_unlink_finalizer(mm) return mm except ImportError: def make_memmap(filename, dtype='uint8', mode='r+', offset=0, shape=None, order='C', unlink_on_gc_collect=False): raise NotImplementedError( "'joblib.backports.make_memmap' should not be used " 'if numpy is not installed.') if os.name == 'nt': # https://github.com/joblib/joblib/issues/540 access_denied_errors = (5, 13) from os import replace def concurrency_safe_rename(src, dst): """Renames ``src`` into ``dst`` overwriting ``dst`` if it exists. On Windows os.replace can yield permission errors if executed by two different processes. """ max_sleep_time = 1 total_sleep_time = 0 sleep_time = 0.001 while total_sleep_time < max_sleep_time: try: replace(src, dst) break except Exception as exc: if getattr(exc, 'winerror', None) in access_denied_errors: time.sleep(sleep_time) total_sleep_time += sleep_time sleep_time *= 2 else: raise else: raise else: from os import replace as concurrency_safe_rename # noqa joblib-1.3.2/joblib/compressor.py000066400000000000000000000464701446465525000170130ustar00rootroot00000000000000"""Classes and functions for managing compressors.""" import io import zlib from joblib.backports import LooseVersion try: from threading import RLock except ImportError: from dummy_threading import RLock try: import bz2 except ImportError: bz2 = None try: import lz4 from lz4.frame import LZ4FrameFile except ImportError: lz4 = None try: import lzma except ImportError: lzma = None LZ4_NOT_INSTALLED_ERROR = ('LZ4 is not installed. Install it with pip: ' 'https://python-lz4.readthedocs.io/') # Registered compressors _COMPRESSORS = {} # Magic numbers of supported compression file formats. _ZFILE_PREFIX = b'ZF' # used with pickle files created before 0.9.3. _ZLIB_PREFIX = b'\x78' _GZIP_PREFIX = b'\x1f\x8b' _BZ2_PREFIX = b'BZ' _XZ_PREFIX = b'\xfd\x37\x7a\x58\x5a' _LZMA_PREFIX = b'\x5d\x00' _LZ4_PREFIX = b'\x04\x22\x4D\x18' def register_compressor(compressor_name, compressor, force=False): """Register a new compressor. Parameters ---------- compressor_name: str. The name of the compressor. compressor: CompressorWrapper An instance of a 'CompressorWrapper'. """ global _COMPRESSORS if not isinstance(compressor_name, str): raise ValueError("Compressor name should be a string, " "'{}' given.".format(compressor_name)) if not isinstance(compressor, CompressorWrapper): raise ValueError("Compressor should implement the CompressorWrapper " "interface, '{}' given.".format(compressor)) if (compressor.fileobj_factory is not None and (not hasattr(compressor.fileobj_factory, 'read') or not hasattr(compressor.fileobj_factory, 'write') or not hasattr(compressor.fileobj_factory, 'seek') or not hasattr(compressor.fileobj_factory, 'tell'))): raise ValueError("Compressor 'fileobj_factory' attribute should " "implement the file object interface, '{}' given." .format(compressor.fileobj_factory)) if compressor_name in _COMPRESSORS and not force: raise ValueError("Compressor '{}' already registered." .format(compressor_name)) _COMPRESSORS[compressor_name] = compressor class CompressorWrapper(): """A wrapper around a compressor file object. Attributes ---------- obj: a file-like object The object must implement the buffer interface and will be used internally to compress/decompress the data. prefix: bytestring A bytestring corresponding to the magic number that identifies the file format associated to the compressor. extension: str The file extension used to automatically select this compressor during a dump to a file. """ def __init__(self, obj, prefix=b'', extension=''): self.fileobj_factory = obj self.prefix = prefix self.extension = extension def compressor_file(self, fileobj, compresslevel=None): """Returns an instance of a compressor file object.""" if compresslevel is None: return self.fileobj_factory(fileobj, 'wb') else: return self.fileobj_factory(fileobj, 'wb', compresslevel=compresslevel) def decompressor_file(self, fileobj): """Returns an instance of a decompressor file object.""" return self.fileobj_factory(fileobj, 'rb') class BZ2CompressorWrapper(CompressorWrapper): prefix = _BZ2_PREFIX extension = '.bz2' def __init__(self): if bz2 is not None: self.fileobj_factory = bz2.BZ2File else: self.fileobj_factory = None def _check_versions(self): if bz2 is None: raise ValueError('bz2 module is not compiled on your python ' 'standard library.') def compressor_file(self, fileobj, compresslevel=None): """Returns an instance of a compressor file object.""" self._check_versions() if compresslevel is None: return self.fileobj_factory(fileobj, 'wb') else: return self.fileobj_factory(fileobj, 'wb', compresslevel=compresslevel) def decompressor_file(self, fileobj): """Returns an instance of a decompressor file object.""" self._check_versions() fileobj = self.fileobj_factory(fileobj, 'rb') return fileobj class LZMACompressorWrapper(CompressorWrapper): prefix = _LZMA_PREFIX extension = '.lzma' _lzma_format_name = 'FORMAT_ALONE' def __init__(self): if lzma is not None: self.fileobj_factory = lzma.LZMAFile self._lzma_format = getattr(lzma, self._lzma_format_name) else: self.fileobj_factory = None def _check_versions(self): if lzma is None: raise ValueError('lzma module is not compiled on your python ' 'standard library.') def compressor_file(self, fileobj, compresslevel=None): """Returns an instance of a compressor file object.""" if compresslevel is None: return self.fileobj_factory(fileobj, 'wb', format=self._lzma_format) else: return self.fileobj_factory(fileobj, 'wb', format=self._lzma_format, preset=compresslevel) def decompressor_file(self, fileobj): """Returns an instance of a decompressor file object.""" return lzma.LZMAFile(fileobj, 'rb') class XZCompressorWrapper(LZMACompressorWrapper): prefix = _XZ_PREFIX extension = '.xz' _lzma_format_name = 'FORMAT_XZ' class LZ4CompressorWrapper(CompressorWrapper): prefix = _LZ4_PREFIX extension = '.lz4' def __init__(self): if lz4 is not None: self.fileobj_factory = LZ4FrameFile else: self.fileobj_factory = None def _check_versions(self): if lz4 is None: raise ValueError(LZ4_NOT_INSTALLED_ERROR) lz4_version = lz4.__version__ if lz4_version.startswith("v"): lz4_version = lz4_version[1:] if LooseVersion(lz4_version) < LooseVersion('0.19'): raise ValueError(LZ4_NOT_INSTALLED_ERROR) def compressor_file(self, fileobj, compresslevel=None): """Returns an instance of a compressor file object.""" self._check_versions() if compresslevel is None: return self.fileobj_factory(fileobj, 'wb') else: return self.fileobj_factory(fileobj, 'wb', compression_level=compresslevel) def decompressor_file(self, fileobj): """Returns an instance of a decompressor file object.""" self._check_versions() return self.fileobj_factory(fileobj, 'rb') ############################################################################### # base file compression/decompression object definition _MODE_CLOSED = 0 _MODE_READ = 1 _MODE_READ_EOF = 2 _MODE_WRITE = 3 _BUFFER_SIZE = 8192 class BinaryZlibFile(io.BufferedIOBase): """A file object providing transparent zlib (de)compression. TODO python2_drop: is it still needed since we dropped Python 2 support A BinaryZlibFile can act as a wrapper for an existing file object, or refer directly to a named file on disk. Note that BinaryZlibFile provides only a *binary* file interface: data read is returned as bytes, and data to be written should be given as bytes. This object is an adaptation of the BZ2File object and is compatible with versions of python >= 2.7. If filename is a str or bytes object, it gives the name of the file to be opened. Otherwise, it should be a file object, which will be used to read or write the compressed data. mode can be 'rb' for reading (default) or 'wb' for (over)writing If mode is 'wb', compresslevel can be a number between 1 and 9 specifying the level of compression: 1 produces the least compression, and 9 produces the most compression. 3 is the default. """ wbits = zlib.MAX_WBITS def __init__(self, filename, mode="rb", compresslevel=3): # This lock must be recursive, so that BufferedIOBase's # readline(), readlines() and writelines() don't deadlock. self._lock = RLock() self._fp = None self._closefp = False self._mode = _MODE_CLOSED self._pos = 0 self._size = -1 self.compresslevel = compresslevel if not isinstance(compresslevel, int) or not (1 <= compresslevel <= 9): raise ValueError("'compresslevel' must be an integer " "between 1 and 9. You provided 'compresslevel={}'" .format(compresslevel)) if mode == "rb": self._mode = _MODE_READ self._decompressor = zlib.decompressobj(self.wbits) self._buffer = b"" self._buffer_offset = 0 elif mode == "wb": self._mode = _MODE_WRITE self._compressor = zlib.compressobj(self.compresslevel, zlib.DEFLATED, self.wbits, zlib.DEF_MEM_LEVEL, 0) else: raise ValueError("Invalid mode: %r" % (mode,)) if isinstance(filename, str): self._fp = io.open(filename, mode) self._closefp = True elif hasattr(filename, "read") or hasattr(filename, "write"): self._fp = filename else: raise TypeError("filename must be a str or bytes object, " "or a file") def close(self): """Flush and close the file. May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ with self._lock: if self._mode == _MODE_CLOSED: return try: if self._mode in (_MODE_READ, _MODE_READ_EOF): self._decompressor = None elif self._mode == _MODE_WRITE: self._fp.write(self._compressor.flush()) self._compressor = None finally: try: if self._closefp: self._fp.close() finally: self._fp = None self._closefp = False self._mode = _MODE_CLOSED self._buffer = b"" self._buffer_offset = 0 @property def closed(self): """True if this file is closed.""" return self._mode == _MODE_CLOSED def fileno(self): """Return the file descriptor for the underlying file.""" self._check_not_closed() return self._fp.fileno() def seekable(self): """Return whether the file supports seeking.""" return self.readable() and self._fp.seekable() def readable(self): """Return whether the file was opened for reading.""" self._check_not_closed() return self._mode in (_MODE_READ, _MODE_READ_EOF) def writable(self): """Return whether the file was opened for writing.""" self._check_not_closed() return self._mode == _MODE_WRITE # Mode-checking helper functions. def _check_not_closed(self): if self.closed: fname = getattr(self._fp, 'name', None) msg = "I/O operation on closed file" if fname is not None: msg += " {}".format(fname) msg += "." raise ValueError(msg) def _check_can_read(self): if self._mode not in (_MODE_READ, _MODE_READ_EOF): self._check_not_closed() raise io.UnsupportedOperation("File not open for reading") def _check_can_write(self): if self._mode != _MODE_WRITE: self._check_not_closed() raise io.UnsupportedOperation("File not open for writing") def _check_can_seek(self): if self._mode not in (_MODE_READ, _MODE_READ_EOF): self._check_not_closed() raise io.UnsupportedOperation("Seeking is only supported " "on files open for reading") if not self._fp.seekable(): raise io.UnsupportedOperation("The underlying file object " "does not support seeking") # Fill the readahead buffer if it is empty. Returns False on EOF. def _fill_buffer(self): if self._mode == _MODE_READ_EOF: return False # Depending on the input data, our call to the decompressor may not # return any data. In this case, try again after reading another block. while self._buffer_offset == len(self._buffer): try: rawblock = (self._decompressor.unused_data or self._fp.read(_BUFFER_SIZE)) if not rawblock: raise EOFError except EOFError: # End-of-stream marker and end of file. We're good. self._mode = _MODE_READ_EOF self._size = self._pos return False else: self._buffer = self._decompressor.decompress(rawblock) self._buffer_offset = 0 return True # Read data until EOF. # If return_data is false, consume the data without returning it. def _read_all(self, return_data=True): # The loop assumes that _buffer_offset is 0. Ensure that this is true. self._buffer = self._buffer[self._buffer_offset:] self._buffer_offset = 0 blocks = [] while self._fill_buffer(): if return_data: blocks.append(self._buffer) self._pos += len(self._buffer) self._buffer = b"" if return_data: return b"".join(blocks) # Read a block of up to n bytes. # If return_data is false, consume the data without returning it. def _read_block(self, n_bytes, return_data=True): # If we have enough data buffered, return immediately. end = self._buffer_offset + n_bytes if end <= len(self._buffer): data = self._buffer[self._buffer_offset: end] self._buffer_offset = end self._pos += len(data) return data if return_data else None # The loop assumes that _buffer_offset is 0. Ensure that this is true. self._buffer = self._buffer[self._buffer_offset:] self._buffer_offset = 0 blocks = [] while n_bytes > 0 and self._fill_buffer(): if n_bytes < len(self._buffer): data = self._buffer[:n_bytes] self._buffer_offset = n_bytes else: data = self._buffer self._buffer = b"" if return_data: blocks.append(data) self._pos += len(data) n_bytes -= len(data) if return_data: return b"".join(blocks) def read(self, size=-1): """Read up to size uncompressed bytes from the file. If size is negative or omitted, read until EOF is reached. Returns b'' if the file is already at EOF. """ with self._lock: self._check_can_read() if size == 0: return b"" elif size < 0: return self._read_all() else: return self._read_block(size) def readinto(self, b): """Read up to len(b) bytes into b. Returns the number of bytes read (0 for EOF). """ with self._lock: return io.BufferedIOBase.readinto(self, b) def write(self, data): """Write a byte string to the file. Returns the number of uncompressed bytes written, which is always len(data). Note that due to buffering, the file on disk may not reflect the data written until close() is called. """ with self._lock: self._check_can_write() # Convert data type if called by io.BufferedWriter. if isinstance(data, memoryview): data = data.tobytes() compressed = self._compressor.compress(data) self._fp.write(compressed) self._pos += len(data) return len(data) # Rewind the file to the beginning of the data stream. def _rewind(self): self._fp.seek(0, 0) self._mode = _MODE_READ self._pos = 0 self._decompressor = zlib.decompressobj(self.wbits) self._buffer = b"" self._buffer_offset = 0 def seek(self, offset, whence=0): """Change the file position. The new position is specified by offset, relative to the position indicated by whence. Values for whence are: 0: start of stream (default); offset must not be negative 1: current stream position 2: end of stream; offset must not be positive Returns the new file position. Note that seeking is emulated, so depending on the parameters, this operation may be extremely slow. """ with self._lock: self._check_can_seek() # Recalculate offset as an absolute file position. if whence == 0: pass elif whence == 1: offset = self._pos + offset elif whence == 2: # Seeking relative to EOF - we need to know the file's size. if self._size < 0: self._read_all(return_data=False) offset = self._size + offset else: raise ValueError("Invalid value for whence: %s" % (whence,)) # Make it so that offset is the number of bytes to skip forward. if offset < self._pos: self._rewind() else: offset -= self._pos # Read and discard data until we reach the desired position. self._read_block(offset, return_data=False) return self._pos def tell(self): """Return the current file position.""" with self._lock: self._check_not_closed() return self._pos class ZlibCompressorWrapper(CompressorWrapper): def __init__(self): CompressorWrapper.__init__(self, obj=BinaryZlibFile, prefix=_ZLIB_PREFIX, extension='.z') class BinaryGzipFile(BinaryZlibFile): """A file object providing transparent gzip (de)compression. If filename is a str or bytes object, it gives the name of the file to be opened. Otherwise, it should be a file object, which will be used to read or write the compressed data. mode can be 'rb' for reading (default) or 'wb' for (over)writing If mode is 'wb', compresslevel can be a number between 1 and 9 specifying the level of compression: 1 produces the least compression, and 9 produces the most compression. 3 is the default. """ wbits = 31 # zlib compressor/decompressor wbits value for gzip format. class GzipCompressorWrapper(CompressorWrapper): def __init__(self): CompressorWrapper.__init__(self, obj=BinaryGzipFile, prefix=_GZIP_PREFIX, extension='.gz') joblib-1.3.2/joblib/disk.py000066400000000000000000000104451446465525000155420ustar00rootroot00000000000000""" Disk management utilities. """ # Authors: Gael Varoquaux # Lars Buitinck # Copyright (c) 2010 Gael Varoquaux # License: BSD Style, 3 clauses. import os import sys import time import errno import shutil from multiprocessing import util try: WindowsError except NameError: WindowsError = OSError def disk_used(path): """ Return the disk usage in a directory.""" size = 0 for file in os.listdir(path) + ['.']: stat = os.stat(os.path.join(path, file)) if hasattr(stat, 'st_blocks'): size += stat.st_blocks * 512 else: # on some platform st_blocks is not available (e.g., Windows) # approximate by rounding to next multiple of 512 size += (stat.st_size // 512 + 1) * 512 # We need to convert to int to avoid having longs on some systems (we # don't want longs to avoid problems we SQLite) return int(size / 1024.) def memstr_to_bytes(text): """ Convert a memory text to its value in bytes. """ kilo = 1024 units = dict(K=kilo, M=kilo ** 2, G=kilo ** 3) try: size = int(units[text[-1]] * float(text[:-1])) except (KeyError, ValueError) as e: raise ValueError( "Invalid literal for size give: %s (type %s) should be " "alike '10G', '500M', '50K'." % (text, type(text))) from e return size def mkdirp(d): """Ensure directory d exists (like mkdir -p on Unix) No guarantee that the directory is writable. """ try: os.makedirs(d) except OSError as e: if e.errno != errno.EEXIST: raise # if a rmtree operation fails in rm_subdirs, wait for this much time (in secs), # then retry up to RM_SUBDIRS_N_RETRY times. If it still fails, raise the # exception. this mechanism ensures that the sub-process gc have the time to # collect and close the memmaps before we fail. RM_SUBDIRS_RETRY_TIME = 0.1 RM_SUBDIRS_N_RETRY = 10 def rm_subdirs(path, onerror=None): """Remove all subdirectories in this path. The directory indicated by `path` is left in place, and its subdirectories are erased. If onerror is set, it is called to handle the error with arguments (func, path, exc_info) where func is os.listdir, os.remove, or os.rmdir; path is the argument to that function that caused it to fail; and exc_info is a tuple returned by sys.exc_info(). If onerror is None, an exception is raised. """ # NOTE this code is adapted from the one in shutil.rmtree, and is # just as fast names = [] try: names = os.listdir(path) except os.error: if onerror is not None: onerror(os.listdir, path, sys.exc_info()) else: raise for name in names: fullname = os.path.join(path, name) delete_folder(fullname, onerror=onerror) def delete_folder(folder_path, onerror=None, allow_non_empty=True): """Utility function to cleanup a temporary folder if it still exists.""" if os.path.isdir(folder_path): if onerror is not None: shutil.rmtree(folder_path, False, onerror) else: # allow the rmtree to fail once, wait and re-try. # if the error is raised again, fail err_count = 0 while True: files = os.listdir(folder_path) try: if len(files) == 0 or allow_non_empty: shutil.rmtree( folder_path, ignore_errors=False, onerror=None ) util.debug( "Successfully deleted {}".format(folder_path)) break else: raise OSError( "Expected empty folder {} but got {} " "files.".format(folder_path, len(files)) ) except (OSError, WindowsError): err_count += 1 if err_count > RM_SUBDIRS_N_RETRY: # the folder cannot be deleted right now. It maybe # because some temporary files have not been deleted # yet. raise time.sleep(RM_SUBDIRS_RETRY_TIME) joblib-1.3.2/joblib/executor.py000066400000000000000000000120201446465525000164350ustar00rootroot00000000000000"""Utility function to construct a loky.ReusableExecutor with custom pickler. This module provides efficient ways of working with data stored in shared memory with numpy.memmap arrays without inducing any memory copy between the parent and child processes. """ # Author: Thomas Moreau # Copyright: 2017, Thomas Moreau # License: BSD 3 clause from ._memmapping_reducer import get_memmapping_reducers from ._memmapping_reducer import TemporaryResourcesManager from .externals.loky.reusable_executor import _ReusablePoolExecutor _executor_args = None def get_memmapping_executor(n_jobs, **kwargs): return MemmappingExecutor.get_memmapping_executor(n_jobs, **kwargs) class MemmappingExecutor(_ReusablePoolExecutor): @classmethod def get_memmapping_executor(cls, n_jobs, timeout=300, initializer=None, initargs=(), env=None, temp_folder=None, context_id=None, **backend_args): """Factory for ReusableExecutor with automatic memmapping for large numpy arrays. """ global _executor_args # Check if we can reuse the executor here instead of deferring the test # to loky as the reducers are objects that changes at each call. executor_args = backend_args.copy() executor_args.update(env if env else {}) executor_args.update(dict( timeout=timeout, initializer=initializer, initargs=initargs)) reuse = _executor_args is None or _executor_args == executor_args _executor_args = executor_args manager = TemporaryResourcesManager(temp_folder) # reducers access the temporary folder in which to store temporary # pickles through a call to manager.resolve_temp_folder_name. resolving # the folder name dynamically is useful to use different folders across # calls of a same reusable executor job_reducers, result_reducers = get_memmapping_reducers( unlink_on_gc_collect=True, temp_folder_resolver=manager.resolve_temp_folder_name, **backend_args) _executor, executor_is_reused = super().get_reusable_executor( n_jobs, job_reducers=job_reducers, result_reducers=result_reducers, reuse=reuse, timeout=timeout, initializer=initializer, initargs=initargs, env=env ) if not executor_is_reused: # Only set a _temp_folder_manager for new executors. Reused # executors already have a _temporary_folder_manager that must not # be re-assigned like that because it is referenced in various # places in the reducing machinery of the executor. _executor._temp_folder_manager = manager if context_id is not None: # Only register the specified context once we know which manager # the current executor is using, in order to not register an atexit # finalizer twice for the same folder. _executor._temp_folder_manager.register_new_context(context_id) return _executor def terminate(self, kill_workers=False): self.shutdown(kill_workers=kill_workers) # When workers are killed in a brutal manner, they cannot execute the # finalizer of their shared memmaps. The refcount of those memmaps may # be off by an unknown number, so instead of decref'ing them, we force # delete the whole temporary folder, and unregister them. There is no # risk of PermissionError at folder deletion because at this # point, all child processes are dead, so all references to temporary # memmaps are closed. Otherwise, just try to delete as much as possible # with allow_non_empty=True but if we can't, it will be clean up later # on by the resource_tracker. with self._submit_resize_lock: self._temp_folder_manager._clean_temporary_resources( force=kill_workers, allow_non_empty=True ) @property def _temp_folder(self): # Legacy property in tests. could be removed if we refactored the # memmapping tests. SHOULD ONLY BE USED IN TESTS! # We cache this property because it is called late in the tests - at # this point, all context have been unregistered, and # resolve_temp_folder_name raises an error. if getattr(self, '_cached_temp_folder', None) is not None: return self._cached_temp_folder else: self._cached_temp_folder = self._temp_folder_manager.resolve_temp_folder_name() # noqa return self._cached_temp_folder class _TestingMemmappingExecutor(MemmappingExecutor): """Wrapper around ReusableExecutor to ease memmapping testing with Pool and Executor. This is only for testing purposes. """ def apply_async(self, func, args): """Schedule a func to be run""" future = self.submit(func, *args) future.get = future.result return future def map(self, f, *args): return list(super().map(f, *args)) joblib-1.3.2/joblib/externals/000077500000000000000000000000001446465525000162375ustar00rootroot00000000000000joblib-1.3.2/joblib/externals/__init__.py000066400000000000000000000000001446465525000203360ustar00rootroot00000000000000joblib-1.3.2/joblib/externals/cloudpickle/000077500000000000000000000000001446465525000205355ustar00rootroot00000000000000joblib-1.3.2/joblib/externals/cloudpickle/__init__.py000066400000000000000000000004441446465525000226500ustar00rootroot00000000000000from .cloudpickle import * # noqa from .cloudpickle_fast import CloudPickler, dumps, dump # noqa # Conform to the convention used by python serialization libraries, which # expose their Pickler subclass at top-level under the "Pickler" name. Pickler = CloudPickler __version__ = '2.2.0' joblib-1.3.2/joblib/externals/cloudpickle/cloudpickle.py000066400000000000000000001045011446465525000234060ustar00rootroot00000000000000""" This class is defined to override standard pickle functionality The goals of it follow: -Serialize lambdas and nested functions to compiled byte code -Deal with main module correctly -Deal with other non-serializable objects It does not include an unpickler, as standard python unpickling suffices. This module was extracted from the `cloud` package, developed by `PiCloud, Inc. `_. Copyright (c) 2012, Regents of the University of California. Copyright (c) 2009 `PiCloud, Inc. `_. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the University of California, Berkeley nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ import builtins import dis import opcode import platform import sys import types import weakref import uuid import threading import typing import warnings from .compat import pickle from collections import OrderedDict from typing import ClassVar, Generic, Union, Tuple, Callable from pickle import _getattribute from importlib._bootstrap import _find_spec try: # pragma: no branch import typing_extensions as _typing_extensions from typing_extensions import Literal, Final except ImportError: _typing_extensions = Literal = Final = None if sys.version_info >= (3, 8): from types import CellType else: def f(): a = 1 def g(): return a return g CellType = type(f().__closure__[0]) # cloudpickle is meant for inter process communication: we expect all # communicating processes to run the same Python version hence we favor # communication speed over compatibility: DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL # Names of modules whose resources should be treated as dynamic. _PICKLE_BY_VALUE_MODULES = set() # Track the provenance of reconstructed dynamic classes to make it possible to # reconstruct instances from the matching singleton class definition when # appropriate and preserve the usual "isinstance" semantics of Python objects. _DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary() _DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary() _DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock() PYPY = platform.python_implementation() == "PyPy" builtin_code_type = None if PYPY: # builtin-code objects only exist in pypy builtin_code_type = type(float.__new__.__code__) _extract_code_globals_cache = weakref.WeakKeyDictionary() def _get_or_create_tracker_id(class_def): with _DYNAMIC_CLASS_TRACKER_LOCK: class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def) if class_tracker_id is None: class_tracker_id = uuid.uuid4().hex _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id _DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def return class_tracker_id def _lookup_class_or_track(class_tracker_id, class_def): if class_tracker_id is not None: with _DYNAMIC_CLASS_TRACKER_LOCK: class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault( class_tracker_id, class_def) _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id return class_def def register_pickle_by_value(module): """Register a module to make it functions and classes picklable by value. By default, functions and classes that are attributes of an importable module are to be pickled by reference, that is relying on re-importing the attribute from the module at load time. If `register_pickle_by_value(module)` is called, all its functions and classes are subsequently to be pickled by value, meaning that they can be loaded in Python processes where the module is not importable. This is especially useful when developing a module in a distributed execution environment: restarting the client Python process with the new source code is enough: there is no need to re-install the new version of the module on all the worker nodes nor to restart the workers. Note: this feature is considered experimental. See the cloudpickle README.md file for more details and limitations. """ if not isinstance(module, types.ModuleType): raise ValueError( f"Input should be a module object, got {str(module)} instead" ) # In the future, cloudpickle may need a way to access any module registered # for pickling by value in order to introspect relative imports inside # functions pickled by value. (see # https://github.com/cloudpipe/cloudpickle/pull/417#issuecomment-873684633). # This access can be ensured by checking that module is present in # sys.modules at registering time and assuming that it will still be in # there when accessed during pickling. Another alternative would be to # store a weakref to the module. Even though cloudpickle does not implement # this introspection yet, in order to avoid a possible breaking change # later, we still enforce the presence of module inside sys.modules. if module.__name__ not in sys.modules: raise ValueError( f"{module} was not imported correctly, have you used an " f"`import` statement to access it?" ) _PICKLE_BY_VALUE_MODULES.add(module.__name__) def unregister_pickle_by_value(module): """Unregister that the input module should be pickled by value.""" if not isinstance(module, types.ModuleType): raise ValueError( f"Input should be a module object, got {str(module)} instead" ) if module.__name__ not in _PICKLE_BY_VALUE_MODULES: raise ValueError(f"{module} is not registered for pickle by value") else: _PICKLE_BY_VALUE_MODULES.remove(module.__name__) def list_registry_pickle_by_value(): return _PICKLE_BY_VALUE_MODULES.copy() def _is_registered_pickle_by_value(module): module_name = module.__name__ if module_name in _PICKLE_BY_VALUE_MODULES: return True while True: parent_name = module_name.rsplit(".", 1)[0] if parent_name == module_name: break if parent_name in _PICKLE_BY_VALUE_MODULES: return True module_name = parent_name return False def _whichmodule(obj, name): """Find the module an object belongs to. This function differs from ``pickle.whichmodule`` in two ways: - it does not mangle the cases where obj's module is __main__ and obj was not found in any module. - Errors arising during module introspection are ignored, as those errors are considered unwanted side effects. """ if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa # Workaround bug in old Python versions: prior to Python 3.7, # T.__module__ would always be set to "typing" even when the TypeVar T # would be defined in a different module. if name is not None and getattr(typing, name, None) is obj: # Built-in TypeVar defined in typing such as AnyStr return 'typing' else: # User defined or third-party TypeVar: __module__ attribute is # irrelevant, thus trigger a exhaustive search for obj in all # modules. module_name = None else: module_name = getattr(obj, '__module__', None) if module_name is not None: return module_name # Protect the iteration by using a copy of sys.modules against dynamic # modules that trigger imports of other modules upon calls to getattr or # other threads importing at the same time. for module_name, module in sys.modules.copy().items(): # Some modules such as coverage can inject non-module objects inside # sys.modules if ( module_name == '__main__' or module is None or not isinstance(module, types.ModuleType) ): continue try: if _getattribute(module, name)[0] is obj: return module_name except Exception: pass return None def _should_pickle_by_reference(obj, name=None): """Test whether an function or a class should be pickled by reference Pickling by reference means by that the object (typically a function or a class) is an attribute of a module that is assumed to be importable in the target Python environment. Loading will therefore rely on importing the module and then calling `getattr` on it to access the function or class. Pickling by reference is the only option to pickle functions and classes in the standard library. In cloudpickle the alternative option is to pickle by value (for instance for interactively or locally defined functions and classes or for attributes of modules that have been explicitly registered to be pickled by value. """ if isinstance(obj, types.FunctionType) or issubclass(type(obj), type): module_and_name = _lookup_module_and_qualname(obj, name=name) if module_and_name is None: return False module, name = module_and_name return not _is_registered_pickle_by_value(module) elif isinstance(obj, types.ModuleType): # We assume that sys.modules is primarily used as a cache mechanism for # the Python import machinery. Checking if a module has been added in # is sys.modules therefore a cheap and simple heuristic to tell us # whether we can assume that a given module could be imported by name # in another Python process. if _is_registered_pickle_by_value(obj): return False return obj.__name__ in sys.modules else: raise TypeError( "cannot check importability of {} instances".format( type(obj).__name__) ) def _lookup_module_and_qualname(obj, name=None): if name is None: name = getattr(obj, '__qualname__', None) if name is None: # pragma: no cover # This used to be needed for Python 2.7 support but is probably not # needed anymore. However we keep the __name__ introspection in case # users of cloudpickle rely on this old behavior for unknown reasons. name = getattr(obj, '__name__', None) module_name = _whichmodule(obj, name) if module_name is None: # In this case, obj.__module__ is None AND obj was not found in any # imported module. obj is thus treated as dynamic. return None if module_name == "__main__": return None # Note: if module_name is in sys.modules, the corresponding module is # assumed importable at unpickling time. See #357 module = sys.modules.get(module_name, None) if module is None: # The main reason why obj's module would not be imported is that this # module has been dynamically created, using for example # types.ModuleType. The other possibility is that module was removed # from sys.modules after obj was created/imported. But this case is not # supported, as the standard pickle does not support it either. return None try: obj2, parent = _getattribute(module, name) except AttributeError: # obj was not found inside the module it points to return None if obj2 is not obj: return None return module, name def _extract_code_globals(co): """ Find all globals names read or written to by codeblock co """ out_names = _extract_code_globals_cache.get(co) if out_names is None: # We use a dict with None values instead of a set to get a # deterministic order (assuming Python 3.6+) and avoid introducing # non-deterministic pickle bytes as a results. out_names = {name: None for name in _walk_global_ops(co)} # Declaring a function inside another one using the "def ..." # syntax generates a constant code object corresponding to the one # of the nested function's As the nested function may itself need # global variables, we need to introspect its code, extract its # globals, (look for code object in it's co_consts attribute..) and # add the result to code_globals if co.co_consts: for const in co.co_consts: if isinstance(const, types.CodeType): out_names.update(_extract_code_globals(const)) _extract_code_globals_cache[co] = out_names return out_names def _find_imported_submodules(code, top_level_dependencies): """ Find currently imported submodules used by a function. Submodules used by a function need to be detected and referenced for the function to work correctly at depickling time. Because submodules can be referenced as attribute of their parent package (``package.submodule``), we need a special introspection technique that does not rely on GLOBAL-related opcodes to find references of them in a code object. Example: ``` import concurrent.futures import cloudpickle def func(): x = concurrent.futures.ThreadPoolExecutor if __name__ == '__main__': cloudpickle.dumps(func) ``` The globals extracted by cloudpickle in the function's state include the concurrent package, but not its submodule (here, concurrent.futures), which is the module used by func. Find_imported_submodules will detect the usage of concurrent.futures. Saving this module alongside with func will ensure that calling func once depickled does not fail due to concurrent.futures not being imported """ subimports = [] # check if any known dependency is an imported package for x in top_level_dependencies: if (isinstance(x, types.ModuleType) and hasattr(x, '__package__') and x.__package__): # check if the package has any currently loaded sub-imports prefix = x.__name__ + '.' # A concurrent thread could mutate sys.modules, # make sure we iterate over a copy to avoid exceptions for name in list(sys.modules): # Older versions of pytest will add a "None" module to # sys.modules. if name is not None and name.startswith(prefix): # check whether the function can address the sub-module tokens = set(name[len(prefix):].split('.')) if not tokens - set(code.co_names): subimports.append(sys.modules[name]) return subimports def cell_set(cell, value): """Set the value of a closure cell. The point of this function is to set the cell_contents attribute of a cell after its creation. This operation is necessary in case the cell contains a reference to the function the cell belongs to, as when calling the function's constructor ``f = types.FunctionType(code, globals, name, argdefs, closure)``, closure will not be able to contain the yet-to-be-created f. In Python3.7, cell_contents is writeable, so setting the contents of a cell can be done simply using >>> cell.cell_contents = value In earlier Python3 versions, the cell_contents attribute of a cell is read only, but this limitation can be worked around by leveraging the Python 3 ``nonlocal`` keyword. In Python2 however, this attribute is read only, and there is no ``nonlocal`` keyword. For this reason, we need to come up with more complicated hacks to set this attribute. The chosen approach is to create a function with a STORE_DEREF opcode, which sets the content of a closure variable. Typically: >>> def inner(value): ... lambda: cell # the lambda makes cell a closure ... cell = value # cell is a closure, so this triggers a STORE_DEREF (Note that in Python2, A STORE_DEREF can never be triggered from an inner function. The function g for example here >>> def f(var): ... def g(): ... var += 1 ... return g will not modify the closure variable ``var```inplace, but instead try to load a local variable var and increment it. As g does not assign the local variable ``var`` any initial value, calling f(1)() will fail at runtime.) Our objective is to set the value of a given cell ``cell``. So we need to somewhat reference our ``cell`` object into the ``inner`` function so that this object (and not the smoke cell of the lambda function) gets affected by the STORE_DEREF operation. In inner, ``cell`` is referenced as a cell variable (an enclosing variable that is referenced by the inner function). If we create a new function cell_set with the exact same code as ``inner``, but with ``cell`` marked as a free variable instead, the STORE_DEREF will be applied on its closure - ``cell``, which we can specify explicitly during construction! The new cell_set variable thus actually sets the contents of a specified cell! Note: we do not make use of the ``nonlocal`` keyword to set the contents of a cell in early python3 versions to limit possible syntax errors in case test and checker libraries decide to parse the whole file. """ if sys.version_info[:2] >= (3, 7): # pragma: no branch cell.cell_contents = value else: _cell_set = types.FunctionType( _cell_set_template_code, {}, '_cell_set', (), (cell,),) _cell_set(value) def _make_cell_set_template_code(): def _cell_set_factory(value): lambda: cell cell = value co = _cell_set_factory.__code__ _cell_set_template_code = types.CodeType( co.co_argcount, co.co_kwonlyargcount, # Python 3 only argument co.co_nlocals, co.co_stacksize, co.co_flags, co.co_code, co.co_consts, co.co_names, co.co_varnames, co.co_filename, co.co_name, co.co_firstlineno, co.co_lnotab, co.co_cellvars, # co_freevars is initialized with co_cellvars (), # co_cellvars is made empty ) return _cell_set_template_code if sys.version_info[:2] < (3, 7): _cell_set_template_code = _make_cell_set_template_code() # relevant opcodes STORE_GLOBAL = opcode.opmap['STORE_GLOBAL'] DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL'] LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL'] GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL) HAVE_ARGUMENT = dis.HAVE_ARGUMENT EXTENDED_ARG = dis.EXTENDED_ARG _BUILTIN_TYPE_NAMES = {} for k, v in types.__dict__.items(): if type(v) is type: _BUILTIN_TYPE_NAMES[v] = k def _builtin_type(name): if name == "ClassType": # pragma: no cover # Backward compat to load pickle files generated with cloudpickle # < 1.3 even if loading pickle files from older versions is not # officially supported. return type return getattr(types, name) def _walk_global_ops(code): """ Yield referenced name for all global-referencing instructions in *code*. """ for instr in dis.get_instructions(code): op = instr.opcode if op in GLOBAL_OPS: yield instr.argval def _extract_class_dict(cls): """Retrieve a copy of the dict of a class without the inherited methods""" clsdict = dict(cls.__dict__) # copy dict proxy to a dict if len(cls.__bases__) == 1: inherited_dict = cls.__bases__[0].__dict__ else: inherited_dict = {} for base in reversed(cls.__bases__): inherited_dict.update(base.__dict__) to_remove = [] for name, value in clsdict.items(): try: base_value = inherited_dict[name] if value is base_value: to_remove.append(name) except KeyError: pass for name in to_remove: clsdict.pop(name) return clsdict if sys.version_info[:2] < (3, 7): # pragma: no branch def _is_parametrized_type_hint(obj): # This is very cheap but might generate false positives. So try to # narrow it down is good as possible. type_module = getattr(type(obj), '__module__', None) from_typing_extensions = type_module == 'typing_extensions' from_typing = type_module == 'typing' # general typing Constructs is_typing = getattr(obj, '__origin__', None) is not None # typing_extensions.Literal is_literal = ( (getattr(obj, '__values__', None) is not None) and from_typing_extensions ) # typing_extensions.Final is_final = ( (getattr(obj, '__type__', None) is not None) and from_typing_extensions ) # typing.ClassVar is_classvar = ( (getattr(obj, '__type__', None) is not None) and from_typing ) # typing.Union/Tuple for old Python 3.5 is_union = getattr(obj, '__union_params__', None) is not None is_tuple = getattr(obj, '__tuple_params__', None) is not None is_callable = ( getattr(obj, '__result__', None) is not None and getattr(obj, '__args__', None) is not None ) return any((is_typing, is_literal, is_final, is_classvar, is_union, is_tuple, is_callable)) def _create_parametrized_type_hint(origin, args): return origin[args] else: _is_parametrized_type_hint = None _create_parametrized_type_hint = None def parametrized_type_hint_getinitargs(obj): # The distorted type check sematic for typing construct becomes: # ``type(obj) is type(TypeHint)``, which means "obj is a # parametrized TypeHint" if type(obj) is type(Literal): # pragma: no branch initargs = (Literal, obj.__values__) elif type(obj) is type(Final): # pragma: no branch initargs = (Final, obj.__type__) elif type(obj) is type(ClassVar): initargs = (ClassVar, obj.__type__) elif type(obj) is type(Generic): initargs = (obj.__origin__, obj.__args__) elif type(obj) is type(Union): initargs = (Union, obj.__args__) elif type(obj) is type(Tuple): initargs = (Tuple, obj.__args__) elif type(obj) is type(Callable): (*args, result) = obj.__args__ if len(args) == 1 and args[0] is Ellipsis: args = Ellipsis else: args = list(args) initargs = (Callable, (args, result)) else: # pragma: no cover raise pickle.PicklingError( f"Cloudpickle Error: Unknown type {type(obj)}" ) return initargs # Tornado support def is_tornado_coroutine(func): """ Return whether *func* is a Tornado coroutine function. Running coroutines are not supported. """ if 'tornado.gen' not in sys.modules: return False gen = sys.modules['tornado.gen'] if not hasattr(gen, "is_coroutine_function"): # Tornado version is too old return False return gen.is_coroutine_function(func) def _rebuild_tornado_coroutine(func): from tornado import gen return gen.coroutine(func) # including pickles unloading functions in this namespace load = pickle.load loads = pickle.loads def subimport(name): # We cannot do simply: `return __import__(name)`: Indeed, if ``name`` is # the name of a submodule, __import__ will return the top-level root module # of this submodule. For instance, __import__('os.path') returns the `os` # module. __import__(name) return sys.modules[name] def dynamic_subimport(name, vars): mod = types.ModuleType(name) mod.__dict__.update(vars) mod.__dict__['__builtins__'] = builtins.__dict__ return mod def _gen_ellipsis(): return Ellipsis def _gen_not_implemented(): return NotImplemented def _get_cell_contents(cell): try: return cell.cell_contents except ValueError: # sentinel used by ``_fill_function`` which will leave the cell empty return _empty_cell_value def instance(cls): """Create a new instance of a class. Parameters ---------- cls : type The class to create an instance of. Returns ------- instance : cls A new instance of ``cls``. """ return cls() @instance class _empty_cell_value: """sentinel for empty closures """ @classmethod def __reduce__(cls): return cls.__name__ def _fill_function(*args): """Fills in the rest of function data into the skeleton function object The skeleton itself is create by _make_skel_func(). """ if len(args) == 2: func = args[0] state = args[1] elif len(args) == 5: # Backwards compat for cloudpickle v0.4.0, after which the `module` # argument was introduced func = args[0] keys = ['globals', 'defaults', 'dict', 'closure_values'] state = dict(zip(keys, args[1:])) elif len(args) == 6: # Backwards compat for cloudpickle v0.4.1, after which the function # state was passed as a dict to the _fill_function it-self. func = args[0] keys = ['globals', 'defaults', 'dict', 'module', 'closure_values'] state = dict(zip(keys, args[1:])) else: raise ValueError(f'Unexpected _fill_value arguments: {args!r}') # - At pickling time, any dynamic global variable used by func is # serialized by value (in state['globals']). # - At unpickling time, func's __globals__ attribute is initialized by # first retrieving an empty isolated namespace that will be shared # with other functions pickled from the same original module # by the same CloudPickler instance and then updated with the # content of state['globals'] to populate the shared isolated # namespace with all the global variables that are specifically # referenced for this function. func.__globals__.update(state['globals']) func.__defaults__ = state['defaults'] func.__dict__ = state['dict'] if 'annotations' in state: func.__annotations__ = state['annotations'] if 'doc' in state: func.__doc__ = state['doc'] if 'name' in state: func.__name__ = state['name'] if 'module' in state: func.__module__ = state['module'] if 'qualname' in state: func.__qualname__ = state['qualname'] if 'kwdefaults' in state: func.__kwdefaults__ = state['kwdefaults'] # _cloudpickle_subimports is a set of submodules that must be loaded for # the pickled function to work correctly at unpickling time. Now that these # submodules are depickled (hence imported), they can be removed from the # object's state (the object state only served as a reference holder to # these submodules) if '_cloudpickle_submodules' in state: state.pop('_cloudpickle_submodules') cells = func.__closure__ if cells is not None: for cell, value in zip(cells, state['closure_values']): if value is not _empty_cell_value: cell_set(cell, value) return func def _make_function(code, globals, name, argdefs, closure): # Setting __builtins__ in globals is needed for nogil CPython. globals["__builtins__"] = __builtins__ return types.FunctionType(code, globals, name, argdefs, closure) def _make_empty_cell(): if False: # trick the compiler into creating an empty cell in our lambda cell = None raise AssertionError('this route should not be executed') return (lambda: cell).__closure__[0] def _make_cell(value=_empty_cell_value): cell = _make_empty_cell() if value is not _empty_cell_value: cell_set(cell, value) return cell def _make_skel_func(code, cell_count, base_globals=None): """ Creates a skeleton function object that contains just the provided code and the correct number of cells in func_closure. All other func attributes (e.g. func_globals) are empty. """ # This function is deprecated and should be removed in cloudpickle 1.7 warnings.warn( "A pickle file created using an old (<=1.4.1) version of cloudpickle " "is currently being loaded. This is not supported by cloudpickle and " "will break in cloudpickle 1.7", category=UserWarning ) # This is backward-compatibility code: for cloudpickle versions between # 0.5.4 and 0.7, base_globals could be a string or None. base_globals # should now always be a dictionary. if base_globals is None or isinstance(base_globals, str): base_globals = {} base_globals['__builtins__'] = __builtins__ closure = ( tuple(_make_empty_cell() for _ in range(cell_count)) if cell_count >= 0 else None ) return types.FunctionType(code, base_globals, None, None, closure) def _make_skeleton_class(type_constructor, name, bases, type_kwargs, class_tracker_id, extra): """Build dynamic class with an empty __dict__ to be filled once memoized If class_tracker_id is not None, try to lookup an existing class definition matching that id. If none is found, track a newly reconstructed class definition under that id so that other instances stemming from the same class id will also reuse this class definition. The "extra" variable is meant to be a dict (or None) that can be used for forward compatibility shall the need arise. """ skeleton_class = types.new_class( name, bases, {'metaclass': type_constructor}, lambda ns: ns.update(type_kwargs) ) return _lookup_class_or_track(class_tracker_id, skeleton_class) def _rehydrate_skeleton_class(skeleton_class, class_dict): """Put attributes from `class_dict` back on `skeleton_class`. See CloudPickler.save_dynamic_class for more info. """ registry = None for attrname, attr in class_dict.items(): if attrname == "_abc_impl": registry = attr else: setattr(skeleton_class, attrname, attr) if registry is not None: for subclass in registry: skeleton_class.register(subclass) return skeleton_class def _make_skeleton_enum(bases, name, qualname, members, module, class_tracker_id, extra): """Build dynamic enum with an empty __dict__ to be filled once memoized The creation of the enum class is inspired by the code of EnumMeta._create_. If class_tracker_id is not None, try to lookup an existing enum definition matching that id. If none is found, track a newly reconstructed enum definition under that id so that other instances stemming from the same class id will also reuse this enum definition. The "extra" variable is meant to be a dict (or None) that can be used for forward compatibility shall the need arise. """ # enums always inherit from their base Enum class at the last position in # the list of base classes: enum_base = bases[-1] metacls = enum_base.__class__ classdict = metacls.__prepare__(name, bases) for member_name, member_value in members.items(): classdict[member_name] = member_value enum_class = metacls.__new__(metacls, name, bases, classdict) enum_class.__module__ = module enum_class.__qualname__ = qualname return _lookup_class_or_track(class_tracker_id, enum_class) def _make_typevar(name, bound, constraints, covariant, contravariant, class_tracker_id): tv = typing.TypeVar( name, *constraints, bound=bound, covariant=covariant, contravariant=contravariant ) if class_tracker_id is not None: return _lookup_class_or_track(class_tracker_id, tv) else: # pragma: nocover # Only for Python 3.5.3 compat. return tv def _decompose_typevar(obj): return ( obj.__name__, obj.__bound__, obj.__constraints__, obj.__covariant__, obj.__contravariant__, _get_or_create_tracker_id(obj), ) def _typevar_reduce(obj): # TypeVar instances require the module information hence why we # are not using the _should_pickle_by_reference directly module_and_name = _lookup_module_and_qualname(obj, name=obj.__name__) if module_and_name is None: return (_make_typevar, _decompose_typevar(obj)) elif _is_registered_pickle_by_value(module_and_name[0]): return (_make_typevar, _decompose_typevar(obj)) return (getattr, module_and_name) def _get_bases(typ): if '__orig_bases__' in getattr(typ, '__dict__', {}): # For generic types (see PEP 560) # Note that simply checking `hasattr(typ, '__orig_bases__')` is not # correct. Subclasses of a fully-parameterized generic class does not # have `__orig_bases__` defined, but `hasattr(typ, '__orig_bases__')` # will return True because it's defined in the base class. bases_attr = '__orig_bases__' else: # For regular class objects bases_attr = '__bases__' return getattr(typ, bases_attr) def _make_dict_keys(obj, is_ordered=False): if is_ordered: return OrderedDict.fromkeys(obj).keys() else: return dict.fromkeys(obj).keys() def _make_dict_values(obj, is_ordered=False): if is_ordered: return OrderedDict((i, _) for i, _ in enumerate(obj)).values() else: return {i: _ for i, _ in enumerate(obj)}.values() def _make_dict_items(obj, is_ordered=False): if is_ordered: return OrderedDict(obj).items() else: return obj.items() joblib-1.3.2/joblib/externals/cloudpickle/cloudpickle_fast.py000066400000000000000000001025021446465525000244220ustar00rootroot00000000000000""" New, fast version of the CloudPickler. This new CloudPickler class can now extend the fast C Pickler instead of the previous Python implementation of the Pickler class. Because this functionality is only available for Python versions 3.8+, a lot of backward-compatibility code is also removed. Note that the C Pickler subclassing API is CPython-specific. Therefore, some guards present in cloudpickle.py that were written to handle PyPy specificities are not present in cloudpickle_fast.py """ import _collections_abc import abc import copyreg import io import itertools import logging import sys import struct import types import weakref import typing from enum import Enum from collections import ChainMap, OrderedDict from .compat import pickle, Pickler from .cloudpickle import ( _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, _find_imported_submodules, _get_cell_contents, _should_pickle_by_reference, _builtin_type, _get_or_create_tracker_id, _make_skeleton_class, _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType, _is_parametrized_type_hint, PYPY, cell_set, parametrized_type_hint_getinitargs, _create_parametrized_type_hint, builtin_code_type, _make_dict_keys, _make_dict_values, _make_dict_items, _make_function, ) if pickle.HIGHEST_PROTOCOL >= 5: # Shorthands similar to pickle.dump/pickle.dumps def dump(obj, file, protocol=None, buffer_callback=None): """Serialize obj as bytes streamed into file protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed between processes running the same Python version. Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. """ CloudPickler( file, protocol=protocol, buffer_callback=buffer_callback ).dump(obj) def dumps(obj, protocol=None, buffer_callback=None): """Serialize obj as a string of bytes allocated in memory protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed between processes running the same Python version. Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. """ with io.BytesIO() as file: cp = CloudPickler( file, protocol=protocol, buffer_callback=buffer_callback ) cp.dump(obj) return file.getvalue() else: # Shorthands similar to pickle.dump/pickle.dumps def dump(obj, file, protocol=None): """Serialize obj as bytes streamed into file protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed between processes running the same Python version. Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. """ CloudPickler(file, protocol=protocol).dump(obj) def dumps(obj, protocol=None): """Serialize obj as a string of bytes allocated in memory protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed between processes running the same Python version. Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. """ with io.BytesIO() as file: cp = CloudPickler(file, protocol=protocol) cp.dump(obj) return file.getvalue() load, loads = pickle.load, pickle.loads # COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS # ------------------------------------------------- def _class_getnewargs(obj): type_kwargs = {} if "__slots__" in obj.__dict__: type_kwargs["__slots__"] = obj.__slots__ __dict__ = obj.__dict__.get('__dict__', None) if isinstance(__dict__, property): type_kwargs['__dict__'] = __dict__ return (type(obj), obj.__name__, _get_bases(obj), type_kwargs, _get_or_create_tracker_id(obj), None) def _enum_getnewargs(obj): members = {e.name: e.value for e in obj} return (obj.__bases__, obj.__name__, obj.__qualname__, members, obj.__module__, _get_or_create_tracker_id(obj), None) # COLLECTION OF OBJECTS RECONSTRUCTORS # ------------------------------------ def _file_reconstructor(retval): return retval # COLLECTION OF OBJECTS STATE GETTERS # ----------------------------------- def _function_getstate(func): # - Put func's dynamic attributes (stored in func.__dict__) in state. These # attributes will be restored at unpickling time using # f.__dict__.update(state) # - Put func's members into slotstate. Such attributes will be restored at # unpickling time by iterating over slotstate and calling setattr(func, # slotname, slotvalue) slotstate = { "__name__": func.__name__, "__qualname__": func.__qualname__, "__annotations__": func.__annotations__, "__kwdefaults__": func.__kwdefaults__, "__defaults__": func.__defaults__, "__module__": func.__module__, "__doc__": func.__doc__, "__closure__": func.__closure__, } f_globals_ref = _extract_code_globals(func.__code__) f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in func.__globals__} closure_values = ( list(map(_get_cell_contents, func.__closure__)) if func.__closure__ is not None else () ) # Extract currently-imported submodules used by func. Storing these modules # in a smoke _cloudpickle_subimports attribute of the object's state will # trigger the side effect of importing these modules at unpickling time # (which is necessary for func to work correctly once depickled) slotstate["_cloudpickle_submodules"] = _find_imported_submodules( func.__code__, itertools.chain(f_globals.values(), closure_values)) slotstate["__globals__"] = f_globals state = func.__dict__ return state, slotstate def _class_getstate(obj): clsdict = _extract_class_dict(obj) clsdict.pop('__weakref__', None) if issubclass(type(obj), abc.ABCMeta): # If obj is an instance of an ABCMeta subclass, don't pickle the # cache/negative caches populated during isinstance/issubclass # checks, but pickle the list of registered subclasses of obj. clsdict.pop('_abc_cache', None) clsdict.pop('_abc_negative_cache', None) clsdict.pop('_abc_negative_cache_version', None) registry = clsdict.pop('_abc_registry', None) if registry is None: # in Python3.7+, the abc caches and registered subclasses of a # class are bundled into the single _abc_impl attribute clsdict.pop('_abc_impl', None) (registry, _, _, _) = abc._get_dump(obj) clsdict["_abc_impl"] = [subclass_weakref() for subclass_weakref in registry] else: # In the above if clause, registry is a set of weakrefs -- in # this case, registry is a WeakSet clsdict["_abc_impl"] = [type_ for type_ in registry] if "__slots__" in clsdict: # pickle string length optimization: member descriptors of obj are # created automatically from obj's __slots__ attribute, no need to # save them in obj's state if isinstance(obj.__slots__, str): clsdict.pop(obj.__slots__) else: for k in obj.__slots__: clsdict.pop(k, None) clsdict.pop('__dict__', None) # unpicklable property object return (clsdict, {}) def _enum_getstate(obj): clsdict, slotstate = _class_getstate(obj) members = {e.name: e.value for e in obj} # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: # Those attributes are already handled by the metaclass. for attrname in ["_generate_next_value_", "_member_names_", "_member_map_", "_member_type_", "_value2member_map_"]: clsdict.pop(attrname, None) for member in members: clsdict.pop(member) # Special handling of Enum subclasses return clsdict, slotstate # COLLECTIONS OF OBJECTS REDUCERS # ------------------------------- # A reducer is a function taking a single argument (obj), and that returns a # tuple with all the necessary data to re-construct obj. Apart from a few # exceptions (list, dict, bytes, int, etc.), a reducer is necessary to # correctly pickle an object. # While many built-in objects (Exceptions objects, instances of the "object" # class, etc), are shipped with their own built-in reducer (invoked using # obj.__reduce__), some do not. The following methods were created to "fill # these holes". def _code_reduce(obj): """codeobject reducer""" # If you are not sure about the order of arguments, take a look at help # of the specific type from types, for example: # >>> from types import CodeType # >>> help(CodeType) if hasattr(obj, "co_exceptiontable"): # pragma: no branch # Python 3.11 and later: there are some new attributes # related to the enhanced exceptions. args = ( obj.co_argcount, obj.co_posonlyargcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, obj.co_freevars, obj.co_cellvars, ) elif hasattr(obj, "co_linetable"): # pragma: no branch # Python 3.10 and later: obj.co_lnotab is deprecated and constructor # expects obj.co_linetable instead. args = ( obj.co_argcount, obj.co_posonlyargcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_linetable, obj.co_freevars, obj.co_cellvars ) elif hasattr(obj, "co_nmeta"): # pragma: no cover # "nogil" Python: modified attributes from 3.9 args = ( obj.co_argcount, obj.co_posonlyargcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_framesize, obj.co_ndefaultargs, obj.co_nmeta, obj.co_flags, obj.co_code, obj.co_consts, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_exc_handlers, obj.co_jump_table, obj.co_freevars, obj.co_cellvars, obj.co_free2reg, obj.co_cell2reg ) elif hasattr(obj, "co_posonlyargcount"): # Backward compat for 3.9 and older args = ( obj.co_argcount, obj.co_posonlyargcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars ) else: # Backward compat for even older versions of Python args = ( obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars ) return types.CodeType, args def _cell_reduce(obj): """Cell (containing values of a function's free variables) reducer""" try: obj.cell_contents except ValueError: # cell is empty return _make_empty_cell, () else: return _make_cell, (obj.cell_contents, ) def _classmethod_reduce(obj): orig_func = obj.__func__ return type(obj), (orig_func,) def _file_reduce(obj): """Save a file""" import io if not hasattr(obj, "name") or not hasattr(obj, "mode"): raise pickle.PicklingError( "Cannot pickle files that do not map to an actual file" ) if obj is sys.stdout: return getattr, (sys, "stdout") if obj is sys.stderr: return getattr, (sys, "stderr") if obj is sys.stdin: raise pickle.PicklingError("Cannot pickle standard input") if obj.closed: raise pickle.PicklingError("Cannot pickle closed files") if hasattr(obj, "isatty") and obj.isatty(): raise pickle.PicklingError( "Cannot pickle files that map to tty objects" ) if "r" not in obj.mode and "+" not in obj.mode: raise pickle.PicklingError( "Cannot pickle files that are not opened for reading: %s" % obj.mode ) name = obj.name retval = io.StringIO() try: # Read the whole file curloc = obj.tell() obj.seek(0) contents = obj.read() obj.seek(curloc) except IOError as e: raise pickle.PicklingError( "Cannot pickle file %s as it cannot be read" % name ) from e retval.write(contents) retval.seek(curloc) retval.name = name return _file_reconstructor, (retval,) def _getset_descriptor_reduce(obj): return getattr, (obj.__objclass__, obj.__name__) def _mappingproxy_reduce(obj): return types.MappingProxyType, (dict(obj),) def _memoryview_reduce(obj): return bytes, (obj.tobytes(),) def _module_reduce(obj): if _should_pickle_by_reference(obj): return subimport, (obj.__name__,) else: # Some external libraries can populate the "__builtins__" entry of a # module's `__dict__` with unpicklable objects (see #316). For that # reason, we do not attempt to pickle the "__builtins__" entry, and # restore a default value for it at unpickling time. state = obj.__dict__.copy() state.pop('__builtins__', None) return dynamic_subimport, (obj.__name__, state) def _method_reduce(obj): return (types.MethodType, (obj.__func__, obj.__self__)) def _logger_reduce(obj): return logging.getLogger, (obj.name,) def _root_logger_reduce(obj): return logging.getLogger, () def _property_reduce(obj): return property, (obj.fget, obj.fset, obj.fdel, obj.__doc__) def _weakset_reduce(obj): return weakref.WeakSet, (list(obj),) def _dynamic_class_reduce(obj): """ Save a class that can't be stored as module global. This method is used to serialize classes that are defined inside functions, or that otherwise can't be serialized as attribute lookups from global modules. """ if Enum is not None and issubclass(obj, Enum): return ( _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj), None, None, _class_setstate ) else: return ( _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj), None, None, _class_setstate ) def _class_reduce(obj): """Select the reducer depending on the dynamic nature of the class obj""" if obj is type(None): # noqa return type, (None,) elif obj is type(Ellipsis): return type, (Ellipsis,) elif obj is type(NotImplemented): return type, (NotImplemented,) elif obj in _BUILTIN_TYPE_NAMES: return _builtin_type, (_BUILTIN_TYPE_NAMES[obj],) elif not _should_pickle_by_reference(obj): return _dynamic_class_reduce(obj) return NotImplemented def _dict_keys_reduce(obj): # Safer not to ship the full dict as sending the rest might # be unintended and could potentially cause leaking of # sensitive information return _make_dict_keys, (list(obj), ) def _dict_values_reduce(obj): # Safer not to ship the full dict as sending the rest might # be unintended and could potentially cause leaking of # sensitive information return _make_dict_values, (list(obj), ) def _dict_items_reduce(obj): return _make_dict_items, (dict(obj), ) def _odict_keys_reduce(obj): # Safer not to ship the full dict as sending the rest might # be unintended and could potentially cause leaking of # sensitive information return _make_dict_keys, (list(obj), True) def _odict_values_reduce(obj): # Safer not to ship the full dict as sending the rest might # be unintended and could potentially cause leaking of # sensitive information return _make_dict_values, (list(obj), True) def _odict_items_reduce(obj): return _make_dict_items, (dict(obj), True) # COLLECTIONS OF OBJECTS STATE SETTERS # ------------------------------------ # state setters are called at unpickling time, once the object is created and # it has to be updated to how it was at unpickling time. def _function_setstate(obj, state): """Update the state of a dynamic function. As __closure__ and __globals__ are readonly attributes of a function, we cannot rely on the native setstate routine of pickle.load_build, that calls setattr on items of the slotstate. Instead, we have to modify them inplace. """ state, slotstate = state obj.__dict__.update(state) obj_globals = slotstate.pop("__globals__") obj_closure = slotstate.pop("__closure__") # _cloudpickle_subimports is a set of submodules that must be loaded for # the pickled function to work correctly at unpickling time. Now that these # submodules are depickled (hence imported), they can be removed from the # object's state (the object state only served as a reference holder to # these submodules) slotstate.pop("_cloudpickle_submodules") obj.__globals__.update(obj_globals) obj.__globals__["__builtins__"] = __builtins__ if obj_closure is not None: for i, cell in enumerate(obj_closure): try: value = cell.cell_contents except ValueError: # cell is empty continue cell_set(obj.__closure__[i], value) for k, v in slotstate.items(): setattr(obj, k, v) def _class_setstate(obj, state): state, slotstate = state registry = None for attrname, attr in state.items(): if attrname == "_abc_impl": registry = attr else: setattr(obj, attrname, attr) if registry is not None: for subclass in registry: obj.register(subclass) return obj class CloudPickler(Pickler): # set of reducers defined and used by cloudpickle (private) _dispatch_table = {} _dispatch_table[classmethod] = _classmethod_reduce _dispatch_table[io.TextIOWrapper] = _file_reduce _dispatch_table[logging.Logger] = _logger_reduce _dispatch_table[logging.RootLogger] = _root_logger_reduce _dispatch_table[memoryview] = _memoryview_reduce _dispatch_table[property] = _property_reduce _dispatch_table[staticmethod] = _classmethod_reduce _dispatch_table[CellType] = _cell_reduce _dispatch_table[types.CodeType] = _code_reduce _dispatch_table[types.GetSetDescriptorType] = _getset_descriptor_reduce _dispatch_table[types.ModuleType] = _module_reduce _dispatch_table[types.MethodType] = _method_reduce _dispatch_table[types.MappingProxyType] = _mappingproxy_reduce _dispatch_table[weakref.WeakSet] = _weakset_reduce _dispatch_table[typing.TypeVar] = _typevar_reduce _dispatch_table[_collections_abc.dict_keys] = _dict_keys_reduce _dispatch_table[_collections_abc.dict_values] = _dict_values_reduce _dispatch_table[_collections_abc.dict_items] = _dict_items_reduce _dispatch_table[type(OrderedDict().keys())] = _odict_keys_reduce _dispatch_table[type(OrderedDict().values())] = _odict_values_reduce _dispatch_table[type(OrderedDict().items())] = _odict_items_reduce _dispatch_table[abc.abstractmethod] = _classmethod_reduce _dispatch_table[abc.abstractclassmethod] = _classmethod_reduce _dispatch_table[abc.abstractstaticmethod] = _classmethod_reduce _dispatch_table[abc.abstractproperty] = _property_reduce dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) # function reducers are defined as instance methods of CloudPickler # objects, as they rely on a CloudPickler attribute (globals_ref) def _dynamic_function_reduce(self, func): """Reduce a function that is not pickleable via attribute lookup.""" newargs = self._function_getnewargs(func) state = _function_getstate(func) return (_make_function, newargs, state, None, None, _function_setstate) def _function_reduce(self, obj): """Reducer for function objects. If obj is a top-level attribute of a file-backed module, this reducer returns NotImplemented, making the CloudPickler fallback to traditional _pickle.Pickler routines to save obj. Otherwise, it reduces obj using a custom cloudpickle reducer designed specifically to handle dynamic functions. As opposed to cloudpickle.py, There no special handling for builtin pypy functions because cloudpickle_fast is CPython-specific. """ if _should_pickle_by_reference(obj): return NotImplemented else: return self._dynamic_function_reduce(obj) def _function_getnewargs(self, func): code = func.__code__ # base_globals represents the future global namespace of func at # unpickling time. Looking it up and storing it in # CloudpiPickler.globals_ref allow functions sharing the same globals # at pickling time to also share them once unpickled, at one condition: # since globals_ref is an attribute of a CloudPickler instance, and # that a new CloudPickler is created each time pickle.dump or # pickle.dumps is called, functions also need to be saved within the # same invocation of cloudpickle.dump/cloudpickle.dumps (for example: # cloudpickle.dumps([f1, f2])). There is no such limitation when using # CloudPickler.dump, as long as the multiple invocations are bound to # the same CloudPickler. base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) if base_globals == {}: # Add module attributes used to resolve relative imports # instructions inside func. for k in ["__package__", "__name__", "__path__", "__file__"]: if k in func.__globals__: base_globals[k] = func.__globals__[k] # Do not bind the free variables before the function is created to # avoid infinite recursion. if func.__closure__ is None: closure = None else: closure = tuple( _make_empty_cell() for _ in range(len(code.co_freevars))) return code, base_globals, None, None, closure def dump(self, obj): try: return Pickler.dump(self, obj) except RuntimeError as e: if "recursion" in e.args[0]: msg = ( "Could not pickle object as excessively deep recursion " "required." ) raise pickle.PicklingError(msg) from e else: raise if pickle.HIGHEST_PROTOCOL >= 5: def __init__(self, file, protocol=None, buffer_callback=None): if protocol is None: protocol = DEFAULT_PROTOCOL Pickler.__init__( self, file, protocol=protocol, buffer_callback=buffer_callback ) # map functions __globals__ attribute ids, to ensure that functions # sharing the same global namespace at pickling time also share # their global namespace at unpickling time. self.globals_ref = {} self.proto = int(protocol) else: def __init__(self, file, protocol=None): if protocol is None: protocol = DEFAULT_PROTOCOL Pickler.__init__(self, file, protocol=protocol) # map functions __globals__ attribute ids, to ensure that functions # sharing the same global namespace at pickling time also share # their global namespace at unpickling time. self.globals_ref = {} assert hasattr(self, 'proto') if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: # Pickler is the C implementation of the CPython pickler and therefore # we rely on reduce_override method to customize the pickler behavior. # `CloudPickler.dispatch` is only left for backward compatibility - note # that when using protocol 5, `CloudPickler.dispatch` is not an # extension of `Pickler.dispatch` dictionary, because CloudPickler # subclasses the C-implemented Pickler, which does not expose a # `dispatch` attribute. Earlier versions of the protocol 5 CloudPickler # used `CloudPickler.dispatch` as a class-level attribute storing all # reducers implemented by cloudpickle, but the attribute name was not a # great choice given the meaning of `CloudPickler.dispatch` when # `CloudPickler` extends the pure-python pickler. dispatch = dispatch_table # Implementation of the reducer_override callback, in order to # efficiently serialize dynamic functions and classes by subclassing # the C-implemented Pickler. # TODO: decorrelate reducer_override (which is tied to CPython's # implementation - would it make sense to backport it to pypy? - and # pickle's protocol 5 which is implementation agnostic. Currently, the # availability of both notions coincide on CPython's pickle and the # pickle5 backport, but it may not be the case anymore when pypy # implements protocol 5 def reducer_override(self, obj): """Type-agnostic reducing callback for function and classes. For performance reasons, subclasses of the C _pickle.Pickler class cannot register custom reducers for functions and classes in the dispatch_table. Reducer for such types must instead implemented in the special reducer_override method. Note that method will be called for any object except a few builtin-types (int, lists, dicts etc.), which differs from reducers in the Pickler's dispatch_table, each of them being invoked for objects of a specific type only. This property comes in handy for classes: although most classes are instances of the ``type`` metaclass, some of them can be instances of other custom metaclasses (such as enum.EnumMeta for example). In particular, the metaclass will likely not be known in advance, and thus cannot be special-cased using an entry in the dispatch_table. reducer_override, among other things, allows us to register a reducer that will be called for any class, independently of its type. Notes: * reducer_override has the priority over dispatch_table-registered reducers. * reducer_override can be used to fix other limitations of cloudpickle for other types that suffered from type-specific reducers, such as Exceptions. See https://github.com/cloudpipe/cloudpickle/issues/248 """ if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch return ( _create_parametrized_type_hint, parametrized_type_hint_getinitargs(obj) ) t = type(obj) try: is_anyclass = issubclass(t, type) except TypeError: # t is not a class (old Boost; see SF #502085) is_anyclass = False if is_anyclass: return _class_reduce(obj) elif isinstance(obj, types.FunctionType): return self._function_reduce(obj) else: # fallback to save_global, including the Pickler's # dispatch_table return NotImplemented else: # When reducer_override is not available, hack the pure-Python # Pickler's types.FunctionType and type savers. Note: the type saver # must override Pickler.save_global, because pickle.py contains a # hard-coded call to save_global when pickling meta-classes. dispatch = Pickler.dispatch.copy() def _save_reduce_pickle5(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, obj=None): save = self.save write = self.write self.save_reduce( func, args, state=None, listitems=listitems, dictitems=dictitems, obj=obj ) # backport of the Python 3.8 state_setter pickle operations save(state_setter) save(obj) # simple BINGET opcode as obj is already memoized. save(state) write(pickle.TUPLE2) # Trigger a state_setter(obj, state) function call. write(pickle.REDUCE) # The purpose of state_setter is to carry-out an # inplace modification of obj. We do not care about what the # method might return, so its output is eventually removed from # the stack. write(pickle.POP) def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj is type(None): # noqa return self.save_reduce(type, (None,), obj=obj) elif obj is type(Ellipsis): return self.save_reduce(type, (Ellipsis,), obj=obj) elif obj is type(NotImplemented): return self.save_reduce(type, (NotImplemented,), obj=obj) elif obj in _BUILTIN_TYPE_NAMES: return self.save_reduce( _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch # Parametrized typing constructs in Python < 3.7 are not # compatible with type checks and ``isinstance`` semantics. For # this reason, it is easier to detect them using a # duck-typing-based check (``_is_parametrized_type_hint``) than # to populate the Pickler's dispatch with type-specific savers. self.save_reduce( _create_parametrized_type_hint, parametrized_type_hint_getinitargs(obj), obj=obj ) elif name is not None: Pickler.save_global(self, obj, name=name) elif not _should_pickle_by_reference(obj, name=name): self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj) else: Pickler.save_global(self, obj, name=name) dispatch[type] = save_global def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ if _should_pickle_by_reference(obj, name=name): return Pickler.save_global(self, obj, name=name) elif PYPY and isinstance(obj.__code__, builtin_code_type): return self.save_pypy_builtin_func(obj) else: return self._save_reduce_pickle5( *self._dynamic_function_reduce(obj), obj=obj ) def save_pypy_builtin_func(self, obj): """Save pypy equivalent of builtin functions. PyPy does not have the concept of builtin-functions. Instead, builtin-functions are simple function instances, but with a builtin-code attribute. Most of the time, builtin functions should be pickled by attribute. But PyPy has flaky support for __qualname__, so some builtin functions such as float.__new__ will be classified as dynamic. For this reason only, we created this special routine. Because builtin-functions are not expected to have closure or globals, there is no additional hack (compared the one already implemented in pickle) to protect ourselves from reference cycles. A simple (reconstructor, newargs, obj.__dict__) tuple is save_reduced. Note also that PyPy improved their support for __qualname__ in v3.6, so this routing should be removed when cloudpickle supports only PyPy 3.6 and later. """ rv = (types.FunctionType, (obj.__code__, {}, obj.__name__, obj.__defaults__, obj.__closure__), obj.__dict__) self.save_reduce(*rv, obj=obj) dispatch[types.FunctionType] = save_function joblib-1.3.2/joblib/externals/cloudpickle/compat.py000066400000000000000000000007741446465525000224020ustar00rootroot00000000000000import sys if sys.version_info < (3, 8): try: import pickle5 as pickle # noqa: F401 from pickle5 import Pickler # noqa: F401 except ImportError: import pickle # noqa: F401 # Use the Python pickler for old CPython versions from pickle import _Pickler as Pickler # noqa: F401 else: import pickle # noqa: F401 # Pickler will the C implementation in CPython and the Python # implementation in PyPy from pickle import Pickler # noqa: F401 joblib-1.3.2/joblib/externals/loky/000077500000000000000000000000001446465525000172155ustar00rootroot00000000000000joblib-1.3.2/joblib/externals/loky/__init__.py000066400000000000000000000021201446465525000213210ustar00rootroot00000000000000r"""The :mod:`loky` module manages a pool of worker that can be re-used across time. It provides a robust and dynamic implementation os the :class:`ProcessPoolExecutor` and a function :func:`get_reusable_executor` which hide the pool management under the hood. """ from concurrent.futures import ( ALL_COMPLETED, FIRST_COMPLETED, FIRST_EXCEPTION, CancelledError, Executor, TimeoutError, as_completed, wait, ) from ._base import Future from .backend.context import cpu_count from .backend.reduction import set_loky_pickler from .reusable_executor import get_reusable_executor from .cloudpickle_wrapper import wrap_non_picklable_objects from .process_executor import BrokenProcessPool, ProcessPoolExecutor __all__ = [ "get_reusable_executor", "cpu_count", "wait", "as_completed", "Future", "Executor", "ProcessPoolExecutor", "BrokenProcessPool", "CancelledError", "TimeoutError", "FIRST_COMPLETED", "FIRST_EXCEPTION", "ALL_COMPLETED", "wrap_non_picklable_objects", "set_loky_pickler", ] __version__ = "3.4.1" joblib-1.3.2/joblib/externals/loky/_base.py000066400000000000000000000020411446465525000206350ustar00rootroot00000000000000############################################################################### # Modification of concurrent.futures.Future # # author: Thomas Moreau and Olivier Grisel # # adapted from concurrent/futures/_base.py (17/02/2017) # * Do not use yield from # * Use old super syntax # # Copyright 2009 Brian Quinlan. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. from concurrent.futures import Future as _BaseFuture from concurrent.futures._base import LOGGER # To make loky._base.Future instances awaitable by concurrent.futures.wait, # derive our custom Future class from _BaseFuture. _invoke_callback is the only # modification made to this class in loky. # TODO investigate why using `concurrent.futures.Future` directly does not # always work in our test suite. class Future(_BaseFuture): def _invoke_callbacks(self): for callback in self._done_callbacks: try: callback(self) except BaseException: LOGGER.exception(f"exception calling callback for {self!r}") joblib-1.3.2/joblib/externals/loky/backend/000077500000000000000000000000001446465525000206045ustar00rootroot00000000000000joblib-1.3.2/joblib/externals/loky/backend/__init__.py000066400000000000000000000004701446465525000227160ustar00rootroot00000000000000import os from multiprocessing import synchronize from .context import get_context def _make_name(): return f"/loky-{os.getpid()}-{next(synchronize.SemLock._rand)}" # monkey patch the name creation for multiprocessing synchronize.SemLock._make_name = staticmethod(_make_name) __all__ = ["get_context"] joblib-1.3.2/joblib/externals/loky/backend/_posix_reduction.py000066400000000000000000000033601446465525000245350ustar00rootroot00000000000000############################################################################### # Extra reducers for Unix based system and connections objects # # author: Thomas Moreau and Olivier Grisel # # adapted from multiprocessing/reduction.py (17/02/2017) # * Add adapted reduction for LokyProcesses and socket/Connection # import os import socket import _socket from multiprocessing.connection import Connection from multiprocessing.context import get_spawning_popen from .reduction import register HAVE_SEND_HANDLE = ( hasattr(socket, "CMSG_LEN") and hasattr(socket, "SCM_RIGHTS") and hasattr(socket.socket, "sendmsg") ) def _mk_inheritable(fd): os.set_inheritable(fd, True) return fd def DupFd(fd): """Return a wrapper for an fd.""" popen_obj = get_spawning_popen() if popen_obj is not None: return popen_obj.DupFd(popen_obj.duplicate_for_child(fd)) elif HAVE_SEND_HANDLE: from multiprocessing import resource_sharer return resource_sharer.DupFd(fd) else: raise TypeError( "Cannot pickle connection object. This object can only be " "passed when spawning a new process" ) def _reduce_socket(s): df = DupFd(s.fileno()) return _rebuild_socket, (df, s.family, s.type, s.proto) def _rebuild_socket(df, family, type, proto): fd = df.detach() return socket.fromfd(fd, family, type, proto) def rebuild_connection(df, readable, writable): fd = df.detach() return Connection(fd, readable, writable) def reduce_connection(conn): df = DupFd(conn.fileno()) return rebuild_connection, (df, conn.readable, conn.writable) register(socket.socket, _reduce_socket) register(_socket.socket, _reduce_socket) register(Connection, reduce_connection) joblib-1.3.2/joblib/externals/loky/backend/_win_reduction.py000066400000000000000000000012531446465525000241670ustar00rootroot00000000000000############################################################################### # Extra reducers for Windows system and connections objects # # author: Thomas Moreau and Olivier Grisel # # adapted from multiprocessing/reduction.py (17/02/2017) # * Add adapted reduction for LokyProcesses and socket/PipeConnection # import socket from multiprocessing import connection from multiprocessing.reduction import _reduce_socket from .reduction import register # register reduction for win32 communication objects register(socket.socket, _reduce_socket) register(connection.Connection, connection.reduce_connection) register(connection.PipeConnection, connection.reduce_pipe_connection) joblib-1.3.2/joblib/externals/loky/backend/context.py000066400000000000000000000325261446465525000226520ustar00rootroot00000000000000############################################################################### # Basic context management with LokyContext # # author: Thomas Moreau and Olivier Grisel # # adapted from multiprocessing/context.py # * Create a context ensuring loky uses only objects that are compatible # * Add LokyContext to the list of context of multiprocessing so loky can be # used with multiprocessing.set_start_method # * Implement a CFS-aware amd physical-core aware cpu_count function. # import os import sys import math import subprocess import traceback import warnings import multiprocessing as mp from multiprocessing import get_context as mp_get_context from multiprocessing.context import BaseContext from .process import LokyProcess, LokyInitMainProcess # Apparently, on older Python versions, loky cannot work 61 workers on Windows # but instead 60: ¯\_(ツ)_/¯ if sys.version_info >= (3, 8): from concurrent.futures.process import _MAX_WINDOWS_WORKERS if sys.version_info < (3, 10): _MAX_WINDOWS_WORKERS = _MAX_WINDOWS_WORKERS - 1 else: # compat for versions before 3.8 which do not define this. _MAX_WINDOWS_WORKERS = 60 START_METHODS = ["loky", "loky_init_main", "spawn"] if sys.platform != "win32": START_METHODS += ["fork", "forkserver"] _DEFAULT_START_METHOD = None # Cache for the number of physical cores to avoid repeating subprocess calls. # It should not change during the lifetime of the program. physical_cores_cache = None def get_context(method=None): # Try to overload the default context method = method or _DEFAULT_START_METHOD or "loky" if method == "fork": # If 'fork' is explicitly requested, warn user about potential issues. warnings.warn( "`fork` start method should not be used with " "`loky` as it does not respect POSIX. Try using " "`spawn` or `loky` instead.", UserWarning, ) try: return mp_get_context(method) except ValueError: raise ValueError( f"Unknown context '{method}'. Value should be in " f"{START_METHODS}." ) def set_start_method(method, force=False): global _DEFAULT_START_METHOD if _DEFAULT_START_METHOD is not None and not force: raise RuntimeError("context has already been set") assert method is None or method in START_METHODS, ( f"'{method}' is not a valid start_method. It should be in " f"{START_METHODS}" ) _DEFAULT_START_METHOD = method def get_start_method(): return _DEFAULT_START_METHOD def cpu_count(only_physical_cores=False): """Return the number of CPUs the current process can use. The returned number of CPUs accounts for: * the number of CPUs in the system, as given by ``multiprocessing.cpu_count``; * the CPU affinity settings of the current process (available on some Unix systems); * Cgroup CPU bandwidth limit (available on Linux only, typically set by docker and similar container orchestration systems); * the value of the LOKY_MAX_CPU_COUNT environment variable if defined. and is given as the minimum of these constraints. If ``only_physical_cores`` is True, return the number of physical cores instead of the number of logical cores (hyperthreading / SMT). Note that this option is not enforced if the number of usable cores is controlled in any other way such as: process affinity, Cgroup restricted CPU bandwidth or the LOKY_MAX_CPU_COUNT environment variable. If the number of physical cores is not found, return the number of logical cores. Note that on Windows, the returned number of CPUs cannot exceed 61 (or 60 for Python < 3.10), see: https://bugs.python.org/issue26903. It is also always larger or equal to 1. """ # Note: os.cpu_count() is allowed to return None in its docstring os_cpu_count = os.cpu_count() or 1 if sys.platform == "win32": # On Windows, attempting to use more than 61 CPUs would result in a # OS-level error. See https://bugs.python.org/issue26903. According to # https://learn.microsoft.com/en-us/windows/win32/procthread/processor-groups # it might be possible to go beyond with a lot of extra work but this # does not look easy. os_cpu_count = min(os_cpu_count, _MAX_WINDOWS_WORKERS) cpu_count_user = _cpu_count_user(os_cpu_count) aggregate_cpu_count = max(min(os_cpu_count, cpu_count_user), 1) if not only_physical_cores: return aggregate_cpu_count if cpu_count_user < os_cpu_count: # Respect user setting return max(cpu_count_user, 1) cpu_count_physical, exception = _count_physical_cores() if cpu_count_physical != "not found": return cpu_count_physical # Fallback to default behavior if exception is not None: # warns only the first time warnings.warn( "Could not find the number of physical cores for the " f"following reason:\n{exception}\n" "Returning the number of logical cores instead. You can " "silence this warning by setting LOKY_MAX_CPU_COUNT to " "the number of cores you want to use." ) traceback.print_tb(exception.__traceback__) return aggregate_cpu_count def _cpu_count_cgroup(os_cpu_count): # Cgroup CPU bandwidth limit available in Linux since 2.6 kernel cpu_max_fname = "/sys/fs/cgroup/cpu.max" cfs_quota_fname = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" cfs_period_fname = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" if os.path.exists(cpu_max_fname): # cgroup v2 # https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html with open(cpu_max_fname) as fh: cpu_quota_us, cpu_period_us = fh.read().strip().split() elif os.path.exists(cfs_quota_fname) and os.path.exists(cfs_period_fname): # cgroup v1 # https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management with open(cfs_quota_fname) as fh: cpu_quota_us = fh.read().strip() with open(cfs_period_fname) as fh: cpu_period_us = fh.read().strip() else: # No Cgroup CPU bandwidth limit (e.g. non-Linux platform) cpu_quota_us = "max" cpu_period_us = 100_000 # unused, for consistency with default values if cpu_quota_us == "max": # No active Cgroup quota on a Cgroup-capable platform return os_cpu_count else: cpu_quota_us = int(cpu_quota_us) cpu_period_us = int(cpu_period_us) if cpu_quota_us > 0 and cpu_period_us > 0: return math.ceil(cpu_quota_us / cpu_period_us) else: # pragma: no cover # Setting a negative cpu_quota_us value is a valid way to disable # cgroup CPU bandwith limits return os_cpu_count def _cpu_count_affinity(os_cpu_count): # Number of available CPUs given affinity settings if hasattr(os, "sched_getaffinity"): try: return len(os.sched_getaffinity(0)) except NotImplementedError: pass # On PyPy and possibly other platforms, os.sched_getaffinity does not exist # or raises NotImplementedError, let's try with the psutil if installed. try: import psutil p = psutil.Process() if hasattr(p, "cpu_affinity"): return len(p.cpu_affinity()) except ImportError: # pragma: no cover if ( sys.platform == "linux" and os.environ.get("LOKY_MAX_CPU_COUNT") is None ): # PyPy does not implement os.sched_getaffinity on Linux which # can cause severe oversubscription problems. Better warn the # user in this particularly pathological case which can wreck # havoc, typically on CI workers. warnings.warn( "Failed to inspect CPU affinity constraints on this system. " "Please install psutil or explictly set LOKY_MAX_CPU_COUNT." ) # This can happen for platforms that do not implement any kind of CPU # infinity such as macOS-based platforms. return os_cpu_count def _cpu_count_user(os_cpu_count): """Number of user defined available CPUs""" cpu_count_affinity = _cpu_count_affinity(os_cpu_count) cpu_count_cgroup = _cpu_count_cgroup(os_cpu_count) # User defined soft-limit passed as a loky specific environment variable. cpu_count_loky = int(os.environ.get("LOKY_MAX_CPU_COUNT", os_cpu_count)) return min(cpu_count_affinity, cpu_count_cgroup, cpu_count_loky) def _count_physical_cores(): """Return a tuple (number of physical cores, exception) If the number of physical cores is found, exception is set to None. If it has not been found, return ("not found", exception). The number of physical cores is cached to avoid repeating subprocess calls. """ exception = None # First check if the value is cached global physical_cores_cache if physical_cores_cache is not None: return physical_cores_cache, exception # Not cached yet, find it try: if sys.platform == "linux": cpu_info = subprocess.run( "lscpu --parse=core".split(), capture_output=True, text=True ) cpu_info = cpu_info.stdout.splitlines() cpu_info = {line for line in cpu_info if not line.startswith("#")} cpu_count_physical = len(cpu_info) elif sys.platform == "win32": cpu_info = subprocess.run( "wmic CPU Get NumberOfCores /Format:csv".split(), capture_output=True, text=True, ) cpu_info = cpu_info.stdout.splitlines() cpu_info = [ l.split(",")[1] for l in cpu_info if (l and l != "Node,NumberOfCores") ] cpu_count_physical = sum(map(int, cpu_info)) elif sys.platform == "darwin": cpu_info = subprocess.run( "sysctl -n hw.physicalcpu".split(), capture_output=True, text=True, ) cpu_info = cpu_info.stdout cpu_count_physical = int(cpu_info) else: raise NotImplementedError(f"unsupported platform: {sys.platform}") # if cpu_count_physical < 1, we did not find a valid value if cpu_count_physical < 1: raise ValueError(f"found {cpu_count_physical} physical cores < 1") except Exception as e: exception = e cpu_count_physical = "not found" # Put the result in cache physical_cores_cache = cpu_count_physical return cpu_count_physical, exception class LokyContext(BaseContext): """Context relying on the LokyProcess.""" _name = "loky" Process = LokyProcess cpu_count = staticmethod(cpu_count) def Queue(self, maxsize=0, reducers=None): """Returns a queue object""" from .queues import Queue return Queue(maxsize, reducers=reducers, ctx=self.get_context()) def SimpleQueue(self, reducers=None): """Returns a queue object""" from .queues import SimpleQueue return SimpleQueue(reducers=reducers, ctx=self.get_context()) if sys.platform != "win32": """For Unix platform, use our custom implementation of synchronize ensuring that we use the loky.backend.resource_tracker to clean-up the semaphores in case of a worker crash. """ def Semaphore(self, value=1): """Returns a semaphore object""" from .synchronize import Semaphore return Semaphore(value=value) def BoundedSemaphore(self, value): """Returns a bounded semaphore object""" from .synchronize import BoundedSemaphore return BoundedSemaphore(value) def Lock(self): """Returns a lock object""" from .synchronize import Lock return Lock() def RLock(self): """Returns a recurrent lock object""" from .synchronize import RLock return RLock() def Condition(self, lock=None): """Returns a condition object""" from .synchronize import Condition return Condition(lock) def Event(self): """Returns an event object""" from .synchronize import Event return Event() class LokyInitMainContext(LokyContext): """Extra context with LokyProcess, which does load the main module This context is used for compatibility in the case ``cloudpickle`` is not present on the running system. This permits to load functions defined in the ``main`` module, using proper safeguards. The declaration of the ``executor`` should be protected by ``if __name__ == "__main__":`` and the functions and variable used from main should be out of this block. This mimics the default behavior of multiprocessing under Windows and the behavior of the ``spawn`` start method on a posix system. For more details, see the end of the following section of python doc https://docs.python.org/3/library/multiprocessing.html#multiprocessing-programming """ _name = "loky_init_main" Process = LokyInitMainProcess # Register loky context so it works with multiprocessing.get_context ctx_loky = LokyContext() mp.context._concrete_contexts["loky"] = ctx_loky mp.context._concrete_contexts["loky_init_main"] = LokyInitMainContext() joblib-1.3.2/joblib/externals/loky/backend/fork_exec.py000066400000000000000000000022421446465525000231230ustar00rootroot00000000000000############################################################################### # Launch a subprocess using forkexec and make sure only the needed fd are # shared in the two process. # # author: Thomas Moreau and Olivier Grisel # import os import sys def close_fds(keep_fds): # pragma: no cover """Close all the file descriptors except those in keep_fds.""" # Make sure to keep stdout and stderr open for logging purpose keep_fds = {*keep_fds, 1, 2} # We try to retrieve all the open fds try: open_fds = {int(fd) for fd in os.listdir("/proc/self/fd")} except FileNotFoundError: import resource max_nfds = resource.getrlimit(resource.RLIMIT_NOFILE)[0] open_fds = {*range(max_nfds)} for i in open_fds - keep_fds: try: os.close(i) except OSError: pass def fork_exec(cmd, keep_fds, env=None): # copy the environment variables to set in the child process env = env or {} child_env = {**os.environ, **env} pid = os.fork() if pid == 0: # pragma: no cover close_fds(keep_fds) os.execve(sys.executable, cmd, child_env) else: return pid joblib-1.3.2/joblib/externals/loky/backend/popen_loky_posix.py000066400000000000000000000127141446465525000245640ustar00rootroot00000000000000############################################################################### # Popen for LokyProcess. # # author: Thomas Moreau and Olivier Grisel # import os import sys import signal import pickle from io import BytesIO from multiprocessing import util, process from multiprocessing.connection import wait from multiprocessing.context import set_spawning_popen from . import reduction, resource_tracker, spawn __all__ = ["Popen"] # # Wrapper for an fd used while launching a process # class _DupFd: def __init__(self, fd): self.fd = reduction._mk_inheritable(fd) def detach(self): return self.fd # # Start child process using subprocess.Popen # class Popen: method = "loky" DupFd = _DupFd def __init__(self, process_obj): sys.stdout.flush() sys.stderr.flush() self.returncode = None self._fds = [] self._launch(process_obj) def duplicate_for_child(self, fd): self._fds.append(fd) return reduction._mk_inheritable(fd) def poll(self, flag=os.WNOHANG): if self.returncode is None: while True: try: pid, sts = os.waitpid(self.pid, flag) except OSError: # Child process not yet created. See #1731717 # e.errno == errno.ECHILD == 10 return None else: break if pid == self.pid: if os.WIFSIGNALED(sts): self.returncode = -os.WTERMSIG(sts) else: assert os.WIFEXITED(sts) self.returncode = os.WEXITSTATUS(sts) return self.returncode def wait(self, timeout=None): if self.returncode is None: if timeout is not None: if not wait([self.sentinel], timeout): return None # This shouldn't block if wait() returned successfully. return self.poll(os.WNOHANG if timeout == 0.0 else 0) return self.returncode def terminate(self): if self.returncode is None: try: os.kill(self.pid, signal.SIGTERM) except ProcessLookupError: pass except OSError: if self.wait(timeout=0.1) is None: raise def _launch(self, process_obj): tracker_fd = resource_tracker._resource_tracker.getfd() fp = BytesIO() set_spawning_popen(self) try: prep_data = spawn.get_preparation_data( process_obj._name, getattr(process_obj, "init_main_module", True), ) reduction.dump(prep_data, fp) reduction.dump(process_obj, fp) finally: set_spawning_popen(None) try: parent_r, child_w = os.pipe() child_r, parent_w = os.pipe() # for fd in self._fds: # _mk_inheritable(fd) cmd_python = [sys.executable] cmd_python += ["-m", self.__module__] cmd_python += ["--process-name", str(process_obj.name)] cmd_python += ["--pipe", str(reduction._mk_inheritable(child_r))] reduction._mk_inheritable(child_w) reduction._mk_inheritable(tracker_fd) self._fds += [child_r, child_w, tracker_fd] if sys.version_info >= (3, 8) and os.name == "posix": mp_tracker_fd = prep_data["mp_tracker_args"]["fd"] self.duplicate_for_child(mp_tracker_fd) from .fork_exec import fork_exec pid = fork_exec(cmd_python, self._fds, env=process_obj.env) util.debug( f"launched python with pid {pid} and cmd:\n{cmd_python}" ) self.sentinel = parent_r method = "getbuffer" if not hasattr(fp, method): method = "getvalue" with os.fdopen(parent_w, "wb") as f: f.write(getattr(fp, method)()) self.pid = pid finally: if parent_r is not None: util.Finalize(self, os.close, (parent_r,)) for fd in (child_r, child_w): if fd is not None: os.close(fd) @staticmethod def thread_is_spawning(): return True if __name__ == "__main__": import argparse parser = argparse.ArgumentParser("Command line parser") parser.add_argument( "--pipe", type=int, required=True, help="File handle for the pipe" ) parser.add_argument( "--process-name", type=str, default=None, help="Identifier for debugging purpose", ) args = parser.parse_args() info = {} exitcode = 1 try: with os.fdopen(args.pipe, "rb") as from_parent: process.current_process()._inheriting = True try: prep_data = pickle.load(from_parent) spawn.prepare(prep_data) process_obj = pickle.load(from_parent) finally: del process.current_process()._inheriting exitcode = process_obj._bootstrap() except Exception: print("\n\n" + "-" * 80) print(f"{args.process_name} failed with traceback: ") print("-" * 80) import traceback print(traceback.format_exc()) print("\n" + "-" * 80) finally: if from_parent is not None: from_parent.close() sys.exit(exitcode) joblib-1.3.2/joblib/externals/loky/backend/popen_loky_win32.py000066400000000000000000000123151446465525000243610ustar00rootroot00000000000000import os import sys import msvcrt import _winapi from pickle import load from multiprocessing import process, util from multiprocessing.context import set_spawning_popen from multiprocessing.popen_spawn_win32 import Popen as _Popen from . import reduction, spawn __all__ = ["Popen"] # # # def _path_eq(p1, p2): return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2) WINENV = hasattr(sys, "_base_executable") and not _path_eq( sys.executable, sys._base_executable ) def _close_handles(*handles): for handle in handles: _winapi.CloseHandle(handle) # # We define a Popen class similar to the one from subprocess, but # whose constructor takes a process object as its argument. # class Popen(_Popen): """ Start a subprocess to run the code of a process object. We differ from cpython implementation with the way we handle environment variables, in order to be able to modify then in the child processes before importing any library, in order to control the number of threads in C-level threadpools. We also use the loky preparation data, in particular to handle main_module inits and the loky resource tracker. """ method = "loky" def __init__(self, process_obj): prep_data = spawn.get_preparation_data( process_obj._name, getattr(process_obj, "init_main_module", True) ) # read end of pipe will be duplicated by the child process # -- see spawn_main() in spawn.py. # # bpo-33929: Previously, the read end of pipe was "stolen" by the child # process, but it leaked a handle if the child process had been # terminated before it could steal the handle from the parent process. rhandle, whandle = _winapi.CreatePipe(None, 0) wfd = msvcrt.open_osfhandle(whandle, 0) cmd = get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle) python_exe = spawn.get_executable() # copy the environment variables to set in the child process child_env = {**os.environ, **process_obj.env} # bpo-35797: When running in a venv, we bypass the redirect # executor and launch our base Python. if WINENV and _path_eq(python_exe, sys.executable): cmd[0] = python_exe = sys._base_executable child_env["__PYVENV_LAUNCHER__"] = sys.executable cmd = " ".join(f'"{x}"' for x in cmd) with open(wfd, "wb") as to_child: # start process try: hp, ht, pid, _ = _winapi.CreateProcess( python_exe, cmd, None, None, False, 0, child_env, None, None, ) _winapi.CloseHandle(ht) except BaseException: _winapi.CloseHandle(rhandle) raise # set attributes of self self.pid = pid self.returncode = None self._handle = hp self.sentinel = int(hp) self.finalizer = util.Finalize( self, _close_handles, (self.sentinel, int(rhandle)) ) # send information to child set_spawning_popen(self) try: reduction.dump(prep_data, to_child) reduction.dump(process_obj, to_child) finally: set_spawning_popen(None) def get_command_line(pipe_handle, parent_pid, **kwds): """Returns prefix of command line used for spawning a child process.""" if getattr(sys, "frozen", False): return [sys.executable, "--multiprocessing-fork", pipe_handle] else: prog = ( "from joblib.externals.loky.backend.popen_loky_win32 import main; " f"main(pipe_handle={pipe_handle}, parent_pid={parent_pid})" ) opts = util._args_from_interpreter_flags() return [ spawn.get_executable(), *opts, "-c", prog, "--multiprocessing-fork", ] def is_forking(argv): """Return whether commandline indicates we are forking.""" if len(argv) >= 2 and argv[1] == "--multiprocessing-fork": return True else: return False def main(pipe_handle, parent_pid=None): """Run code specified by data received over pipe.""" assert is_forking(sys.argv), "Not forking" if parent_pid is not None: source_process = _winapi.OpenProcess( _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE, False, parent_pid ) else: source_process = None new_handle = reduction.duplicate( pipe_handle, source_process=source_process ) fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY) parent_sentinel = source_process with os.fdopen(fd, "rb", closefd=True) as from_parent: process.current_process()._inheriting = True try: preparation_data = load(from_parent) spawn.prepare(preparation_data, parent_sentinel) self = load(from_parent) finally: del process.current_process()._inheriting exitcode = self._bootstrap(parent_sentinel) sys.exit(exitcode) joblib-1.3.2/joblib/externals/loky/backend/process.py000066400000000000000000000037421446465525000226420ustar00rootroot00000000000000############################################################################### # LokyProcess implementation # # authors: Thomas Moreau and Olivier Grisel # # based on multiprocessing/process.py (17/02/2017) # import sys from multiprocessing.context import assert_spawning from multiprocessing.process import BaseProcess class LokyProcess(BaseProcess): _start_method = "loky" def __init__( self, group=None, target=None, name=None, args=(), kwargs={}, daemon=None, init_main_module=False, env=None, ): super().__init__( group=group, target=target, name=name, args=args, kwargs=kwargs, daemon=daemon, ) self.env = {} if env is None else env self.authkey = self.authkey self.init_main_module = init_main_module @staticmethod def _Popen(process_obj): if sys.platform == "win32": from .popen_loky_win32 import Popen else: from .popen_loky_posix import Popen return Popen(process_obj) class LokyInitMainProcess(LokyProcess): _start_method = "loky_init_main" def __init__( self, group=None, target=None, name=None, args=(), kwargs={}, daemon=None, ): super().__init__( group=group, target=target, name=name, args=args, kwargs=kwargs, daemon=daemon, init_main_module=True, ) # # We subclass bytes to avoid accidental transmission of auth keys over network # class AuthenticationKey(bytes): def __reduce__(self): try: assert_spawning(self) except RuntimeError: raise TypeError( "Pickling an AuthenticationKey object is " "disallowed for security reasons" ) return AuthenticationKey, (bytes(self),) joblib-1.3.2/joblib/externals/loky/backend/queues.py000066400000000000000000000162321446465525000224710ustar00rootroot00000000000000############################################################################### # Queue and SimpleQueue implementation for loky # # authors: Thomas Moreau, Olivier Grisel # # based on multiprocessing/queues.py (16/02/2017) # * Add some custom reducers for the Queues/SimpleQueue to tweak the # pickling process. (overload Queue._feed/SimpleQueue.put) # import os import sys import errno import weakref import threading from multiprocessing import util from multiprocessing.queues import ( Full, Queue as mp_Queue, SimpleQueue as mp_SimpleQueue, _sentinel, ) from multiprocessing.context import assert_spawning from .reduction import dumps __all__ = ["Queue", "SimpleQueue", "Full"] class Queue(mp_Queue): def __init__(self, maxsize=0, reducers=None, ctx=None): super().__init__(maxsize=maxsize, ctx=ctx) self._reducers = reducers # Use custom queue set/get state to be able to reduce the custom reducers def __getstate__(self): assert_spawning(self) return ( self._ignore_epipe, self._maxsize, self._reader, self._writer, self._reducers, self._rlock, self._wlock, self._sem, self._opid, ) def __setstate__(self, state): ( self._ignore_epipe, self._maxsize, self._reader, self._writer, self._reducers, self._rlock, self._wlock, self._sem, self._opid, ) = state if sys.version_info >= (3, 9): self._reset() else: self._after_fork() # Overload _start_thread to correctly call our custom _feed def _start_thread(self): util.debug("Queue._start_thread()") # Start thread which transfers data from buffer to pipe self._buffer.clear() self._thread = threading.Thread( target=Queue._feed, args=( self._buffer, self._notempty, self._send_bytes, self._wlock, self._writer.close, self._reducers, self._ignore_epipe, self._on_queue_feeder_error, self._sem, ), name="QueueFeederThread", ) self._thread.daemon = True util.debug("doing self._thread.start()") self._thread.start() util.debug("... done self._thread.start()") # On process exit we will wait for data to be flushed to pipe. # # However, if this process created the queue then all # processes which use the queue will be descendants of this # process. Therefore waiting for the queue to be flushed # is pointless once all the child processes have been joined. created_by_this_process = self._opid == os.getpid() if not self._joincancelled and not created_by_this_process: self._jointhread = util.Finalize( self._thread, Queue._finalize_join, [weakref.ref(self._thread)], exitpriority=-5, ) # Send sentinel to the thread queue object when garbage collected self._close = util.Finalize( self, Queue._finalize_close, [self._buffer, self._notempty], exitpriority=10, ) # Overload the _feed methods to use our custom pickling strategy. @staticmethod def _feed( buffer, notempty, send_bytes, writelock, close, reducers, ignore_epipe, onerror, queue_sem, ): util.debug("starting thread to feed data to pipe") nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel if sys.platform != "win32": wacquire = writelock.acquire wrelease = writelock.release else: wacquire = None while True: try: nacquire() try: if not buffer: nwait() finally: nrelease() try: while True: obj = bpopleft() if obj is sentinel: util.debug("feeder thread got sentinel -- exiting") close() return # serialize the data before acquiring the lock obj_ = dumps(obj, reducers=reducers) if wacquire is None: send_bytes(obj_) else: wacquire() try: send_bytes(obj_) finally: wrelease() # Remove references early to avoid leaking memory del obj, obj_ except IndexError: pass except BaseException as e: if ignore_epipe and getattr(e, "errno", 0) == errno.EPIPE: return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. if util.is_exiting(): util.info(f"error in queue thread: {e}") return else: queue_sem.release() onerror(e, obj) def _on_queue_feeder_error(self, e, obj): """ Private API hook called when feeding data in the background thread raises an exception. For overriding by concurrent.futures. """ import traceback traceback.print_exc() class SimpleQueue(mp_SimpleQueue): def __init__(self, reducers=None, ctx=None): super().__init__(ctx=ctx) # Add possiblity to use custom reducers self._reducers = reducers def close(self): self._reader.close() self._writer.close() # Use custom queue set/get state to be able to reduce the custom reducers def __getstate__(self): assert_spawning(self) return ( self._reader, self._writer, self._reducers, self._rlock, self._wlock, ) def __setstate__(self, state): ( self._reader, self._writer, self._reducers, self._rlock, self._wlock, ) = state # Overload put to use our customizable reducer def put(self, obj): # serialize the data before acquiring the lock obj = dumps(obj, reducers=self._reducers) if self._wlock is None: # writes to a message oriented win32 pipe are atomic self._writer.send_bytes(obj) else: with self._wlock: self._writer.send_bytes(obj) joblib-1.3.2/joblib/externals/loky/backend/reduction.py000066400000000000000000000156271446465525000231650ustar00rootroot00000000000000############################################################################### # Customizable Pickler with some basic reducers # # author: Thomas Moreau # # adapted from multiprocessing/reduction.py (17/02/2017) # * Replace the ForkingPickler with a similar _LokyPickler, # * Add CustomizableLokyPickler to allow customizing pickling process # on the fly. # import copyreg import io import functools import types import sys import os from multiprocessing import util from pickle import loads, HIGHEST_PROTOCOL ############################################################################### # Enable custom pickling in Loky. _dispatch_table = {} def register(type_, reduce_function): _dispatch_table[type_] = reduce_function ############################################################################### # Registers extra pickling routines to improve picklization for loky # make methods picklable def _reduce_method(m): if m.__self__ is None: return getattr, (m.__class__, m.__func__.__name__) else: return getattr, (m.__self__, m.__func__.__name__) class _C: def f(self): pass @classmethod def h(cls): pass register(type(_C().f), _reduce_method) register(type(_C.h), _reduce_method) if not hasattr(sys, "pypy_version_info"): # PyPy uses functions instead of method_descriptors and wrapper_descriptors def _reduce_method_descriptor(m): return getattr, (m.__objclass__, m.__name__) register(type(list.append), _reduce_method_descriptor) register(type(int.__add__), _reduce_method_descriptor) # Make partial func pickable def _reduce_partial(p): return _rebuild_partial, (p.func, p.args, p.keywords or {}) def _rebuild_partial(func, args, keywords): return functools.partial(func, *args, **keywords) register(functools.partial, _reduce_partial) if sys.platform != "win32": from ._posix_reduction import _mk_inheritable # noqa: F401 else: from . import _win_reduction # noqa: F401 # global variable to change the pickler behavior try: from joblib.externals import cloudpickle # noqa: F401 DEFAULT_ENV = "cloudpickle" except ImportError: # If cloudpickle is not present, fallback to pickle DEFAULT_ENV = "pickle" ENV_LOKY_PICKLER = os.environ.get("LOKY_PICKLER", DEFAULT_ENV) _LokyPickler = None _loky_pickler_name = None def set_loky_pickler(loky_pickler=None): global _LokyPickler, _loky_pickler_name if loky_pickler is None: loky_pickler = ENV_LOKY_PICKLER loky_pickler_cls = None # The default loky_pickler is cloudpickle if loky_pickler in ["", None]: loky_pickler = "cloudpickle" if loky_pickler == _loky_pickler_name: return if loky_pickler == "cloudpickle": from joblib.externals.cloudpickle import CloudPickler as loky_pickler_cls else: try: from importlib import import_module module_pickle = import_module(loky_pickler) loky_pickler_cls = module_pickle.Pickler except (ImportError, AttributeError) as e: extra_info = ( "\nThis error occurred while setting loky_pickler to" f" '{loky_pickler}', as required by the env variable " "LOKY_PICKLER or the function set_loky_pickler." ) e.args = (e.args[0] + extra_info,) + e.args[1:] e.msg = e.args[0] raise e util.debug( f"Using '{loky_pickler if loky_pickler else 'cloudpickle'}' for " "serialization." ) class CustomizablePickler(loky_pickler_cls): _loky_pickler_cls = loky_pickler_cls def _set_dispatch_table(self, dispatch_table): for ancestor_class in self._loky_pickler_cls.mro(): dt_attribute = getattr(ancestor_class, "dispatch_table", None) if isinstance(dt_attribute, types.MemberDescriptorType): # Ancestor class (typically _pickle.Pickler) has a # member_descriptor for its "dispatch_table" attribute. Use # it to set the dispatch_table as a member instead of a # dynamic attribute in the __dict__ of the instance, # otherwise it will not be taken into account by the C # implementation of the dump method if a subclass defines a # class-level dispatch_table attribute as was done in # cloudpickle 1.6.0: # https://github.com/joblib/loky/pull/260 dt_attribute.__set__(self, dispatch_table) break # On top of member descriptor set, also use setattr such that code # that directly access self.dispatch_table gets a consistent view # of the same table. self.dispatch_table = dispatch_table def __init__(self, writer, reducers=None, protocol=HIGHEST_PROTOCOL): loky_pickler_cls.__init__(self, writer, protocol=protocol) if reducers is None: reducers = {} if hasattr(self, "dispatch_table"): # Force a copy that we will update without mutating the # any class level defined dispatch_table. loky_dt = dict(self.dispatch_table) else: # Use standard reducers as bases loky_dt = copyreg.dispatch_table.copy() # Register loky specific reducers loky_dt.update(_dispatch_table) # Set the new dispatch table, taking care of the fact that we # need to use the member_descriptor when we inherit from a # subclass of the C implementation of the Pickler base class # with an class level dispatch_table attribute. self._set_dispatch_table(loky_dt) # Register the reducers for type, reduce_func in reducers.items(): self.register(type, reduce_func) def register(self, type, reduce_func): """Attach a reducer function to a given type in the dispatch table.""" self.dispatch_table[type] = reduce_func _LokyPickler = CustomizablePickler _loky_pickler_name = loky_pickler def get_loky_pickler_name(): global _loky_pickler_name return _loky_pickler_name def get_loky_pickler(): global _LokyPickler return _LokyPickler # Set it to its default value set_loky_pickler() def dump(obj, file, reducers=None, protocol=None): """Replacement for pickle.dump() using _LokyPickler.""" global _LokyPickler _LokyPickler(file, reducers=reducers, protocol=protocol).dump(obj) def dumps(obj, reducers=None, protocol=None): global _LokyPickler buf = io.BytesIO() dump(obj, buf, reducers=reducers, protocol=protocol) return buf.getbuffer() __all__ = ["dump", "dumps", "loads", "register", "set_loky_pickler"] if sys.platform == "win32": from multiprocessing.reduction import duplicate __all__ += ["duplicate"] joblib-1.3.2/joblib/externals/loky/backend/resource_tracker.py000066400000000000000000000342421446465525000245250ustar00rootroot00000000000000############################################################################### # Server process to keep track of unlinked resources, like folders and # semaphores and clean them. # # author: Thomas Moreau # # adapted from multiprocessing/semaphore_tracker.py (17/02/2017) # * include custom spawnv_passfds to start the process # * add some VERBOSE logging # # TODO: multiprocessing.resource_tracker was contributed to Python 3.8 so # once loky drops support for Python 3.7 it might be possible to stop # maintaining this loky-specific fork. As a consequence, it might also be # possible to stop maintaining the loky.backend.synchronize fork of # multiprocessing.synchronize. # # On Unix we run a server process which keeps track of unlinked # resources. The server ignores SIGINT and SIGTERM and reads from a # pipe. The resource_tracker implements a reference counting scheme: each time # a Python process anticipates the shared usage of a resource by another # process, it signals the resource_tracker of this shared usage, and in return, # the resource_tracker increments the resource's reference count by 1. # Similarly, when access to a resource is closed by a Python process, the # process notifies the resource_tracker by asking it to decrement the # resource's reference count by 1. When the reference count drops to 0, the # resource_tracker attempts to clean up the underlying resource. # Finally, every other process connected to the resource tracker has a copy of # the writable end of the pipe used to communicate with it, so the resource # tracker gets EOF when all other processes have exited. Then the # resource_tracker process unlinks any remaining leaked resources (with # reference count above 0) # For semaphores, this is important because the system only supports a limited # number of named semaphores, and they will not be automatically removed till # the next reboot. Without this resource tracker process, "killall python" # would probably leave unlinked semaphores. # Note that this behavior differs from CPython's resource_tracker, which only # implements list of shared resources, and not a proper refcounting scheme. # Also, CPython's resource tracker will only attempt to cleanup those shared # resources once all procsses connected to the resouce tracker have exited. import os import shutil import sys import signal import warnings import threading from _multiprocessing import sem_unlink from multiprocessing import util from . import spawn if sys.platform == "win32": import _winapi import msvcrt from multiprocessing.reduction import duplicate __all__ = ["ensure_running", "register", "unregister"] _HAVE_SIGMASK = hasattr(signal, "pthread_sigmask") _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM) _CLEANUP_FUNCS = {"folder": shutil.rmtree, "file": os.unlink} if os.name == "posix": _CLEANUP_FUNCS["semlock"] = sem_unlink VERBOSE = False class ResourceTracker: def __init__(self): self._lock = threading.Lock() self._fd = None self._pid = None def getfd(self): self.ensure_running() return self._fd def ensure_running(self): """Make sure that resource tracker process is running. This can be run from any process. Usually a child process will use the resource created by its parent.""" with self._lock: if self._fd is not None: # resource tracker was launched before, is it still running? if self._check_alive(): # => still alive return # => dead, launch it again os.close(self._fd) if os.name == "posix": try: # At this point, the resource_tracker process has been # killed or crashed. Let's remove the process entry # from the process table to avoid zombie processes. os.waitpid(self._pid, 0) except OSError: # The process was terminated or is a child from an # ancestor of the current process. pass self._fd = None self._pid = None warnings.warn( "resource_tracker: process died unexpectedly, " "relaunching. Some folders/sempahores might " "leak." ) fds_to_pass = [] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: pass r, w = os.pipe() if sys.platform == "win32": _r = duplicate(msvcrt.get_osfhandle(r), inheritable=True) os.close(r) r = _r cmd = f"from {main.__module__} import main; main({r}, {VERBOSE})" try: fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() args = [exe, *util._args_from_interpreter_flags(), "-c", cmd] util.debug(f"launching resource tracker: {args}") # bpo-33613: Register a signal mask that will block the # signals. This signal mask will be inherited by the child # that is going to be spawned and will protect the child from a # race condition that can make the child die before it # registers signal handlers for SIGINT and SIGTERM. The mask is # unregistered after spawning the child. try: if _HAVE_SIGMASK: signal.pthread_sigmask( signal.SIG_BLOCK, _IGNORED_SIGNALS ) pid = spawnv_passfds(exe, args, fds_to_pass) finally: if _HAVE_SIGMASK: signal.pthread_sigmask( signal.SIG_UNBLOCK, _IGNORED_SIGNALS ) except BaseException: os.close(w) raise else: self._fd = w self._pid = pid finally: if sys.platform == "win32": _winapi.CloseHandle(r) else: os.close(r) def _check_alive(self): """Check for the existence of the resource tracker process.""" try: self._send("PROBE", "", "") except BrokenPipeError: return False else: return True def register(self, name, rtype): """Register a named resource, and increment its refcount.""" self.ensure_running() self._send("REGISTER", name, rtype) def unregister(self, name, rtype): """Unregister a named resource with resource tracker.""" self.ensure_running() self._send("UNREGISTER", name, rtype) def maybe_unlink(self, name, rtype): """Decrement the refcount of a resource, and delete it if it hits 0""" self.ensure_running() self._send("MAYBE_UNLINK", name, rtype) def _send(self, cmd, name, rtype): if len(name) > 512: # posix guarantees that writes to a pipe of less than PIPE_BUF # bytes are atomic, and that PIPE_BUF >= 512 raise ValueError("name too long") msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") nbytes = os.write(self._fd, msg) assert nbytes == len(msg) _resource_tracker = ResourceTracker() ensure_running = _resource_tracker.ensure_running register = _resource_tracker.register maybe_unlink = _resource_tracker.maybe_unlink unregister = _resource_tracker.unregister getfd = _resource_tracker.getfd def main(fd, verbose=0): """Run resource tracker.""" # protect the process from ^C and "killall python" etc if verbose: util.log_to_stderr(level=util.DEBUG) signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) for f in (sys.stdin, sys.stdout): try: f.close() except Exception: pass if verbose: util.debug("Main resource tracker is running") registry = {rtype: {} for rtype in _CLEANUP_FUNCS.keys()} try: # keep track of registered/unregistered resources if sys.platform == "win32": fd = msvcrt.open_osfhandle(fd, os.O_RDONLY) with open(fd, "rb") as f: while True: line = f.readline() if line == b"": # EOF break try: splitted = line.strip().decode("ascii").split(":") # name can potentially contain separator symbols (for # instance folders on Windows) cmd, name, rtype = ( splitted[0], ":".join(splitted[1:-1]), splitted[-1], ) if cmd == "PROBE": continue if rtype not in _CLEANUP_FUNCS: raise ValueError( f"Cannot register {name} for automatic cleanup: " f"unknown resource type ({rtype}). Resource type " "should be one of the following: " f"{list(_CLEANUP_FUNCS.keys())}" ) if cmd == "REGISTER": if name not in registry[rtype]: registry[rtype][name] = 1 else: registry[rtype][name] += 1 if verbose: util.debug( "[ResourceTracker] incremented refcount of " f"{rtype} {name} " f"(current {registry[rtype][name]})" ) elif cmd == "UNREGISTER": del registry[rtype][name] if verbose: util.debug( f"[ResourceTracker] unregister {name} {rtype}: " f"registry({len(registry)})" ) elif cmd == "MAYBE_UNLINK": registry[rtype][name] -= 1 if verbose: util.debug( "[ResourceTracker] decremented refcount of " f"{rtype} {name} " f"(current {registry[rtype][name]})" ) if registry[rtype][name] == 0: del registry[rtype][name] try: if verbose: util.debug( f"[ResourceTracker] unlink {name}" ) _CLEANUP_FUNCS[rtype](name) except Exception as e: warnings.warn( f"resource_tracker: {name}: {e!r}" ) else: raise RuntimeError(f"unrecognized command {cmd!r}") except BaseException: try: sys.excepthook(*sys.exc_info()) except BaseException: pass finally: # all processes have terminated; cleanup any remaining resources def _unlink_resources(rtype_registry, rtype): if rtype_registry: try: warnings.warn( "resource_tracker: There appear to be " f"{len(rtype_registry)} leaked {rtype} objects to " "clean up at shutdown" ) except Exception: pass for name in rtype_registry: # For some reason the process which created and registered this # resource has failed to unregister it. Presumably it has # died. We therefore clean it up. try: _CLEANUP_FUNCS[rtype](name) if verbose: util.debug(f"[ResourceTracker] unlink {name}") except Exception as e: warnings.warn(f"resource_tracker: {name}: {e!r}") for rtype, rtype_registry in registry.items(): if rtype == "folder": continue else: _unlink_resources(rtype_registry, rtype) # The default cleanup routine for folders deletes everything inside # those folders recursively, which can include other resources tracked # by the resource tracker). To limit the risk of the resource tracker # attempting to delete twice a resource (once as part of a tracked # folder, and once as a resource), we delete the folders after all # other resource types. if "folder" in registry: _unlink_resources(registry["folder"], "folder") if verbose: util.debug("resource tracker shut down") # # Start a program with only specified fds kept open # def spawnv_passfds(path, args, passfds): passfds = sorted(passfds) if sys.platform != "win32": errpipe_read, errpipe_write = os.pipe() try: from .reduction import _mk_inheritable from .fork_exec import fork_exec _pass = [_mk_inheritable(fd) for fd in passfds] return fork_exec(args, _pass) finally: os.close(errpipe_read) os.close(errpipe_write) else: cmd = " ".join(f'"{x}"' for x in args) try: _, ht, pid, _ = _winapi.CreateProcess( path, cmd, None, None, True, 0, None, None, None ) _winapi.CloseHandle(ht) except BaseException: pass return pid joblib-1.3.2/joblib/externals/loky/backend/spawn.py000066400000000000000000000214021446465525000223050ustar00rootroot00000000000000############################################################################### # Prepares and processes the data to setup the new process environment # # author: Thomas Moreau and Olivier Grisel # # adapted from multiprocessing/spawn.py (17/02/2017) # * Improve logging data # import os import sys import runpy import textwrap import types from multiprocessing import process, util if sys.platform != "win32": WINEXE = False WINSERVICE = False else: import msvcrt from multiprocessing.reduction import duplicate WINEXE = sys.platform == "win32" and getattr(sys, "frozen", False) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") if WINSERVICE: _python_exe = os.path.join(sys.exec_prefix, "python.exe") else: _python_exe = sys.executable def get_executable(): return _python_exe def _check_not_importing_main(): if getattr(process.current_process(), "_inheriting", False): raise RuntimeError( textwrap.dedent( """\ An attempt has been made to start a new process before the current process has finished its bootstrapping phase. This probably means that you are not using fork to start your child processes and you have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce an executable.""" ) ) def get_preparation_data(name, init_main_module=True): """Return info about parent needed by child to unpickle process object.""" _check_not_importing_main() d = dict( log_to_stderr=util._log_to_stderr, authkey=bytes(process.current_process().authkey), name=name, sys_argv=sys.argv, orig_dir=process.ORIGINAL_DIR, dir=os.getcwd(), ) # Send sys_path and make sure the current directory will not be changed d["sys_path"] = [p if p != "" else process.ORIGINAL_DIR for p in sys.path] # Make sure to pass the information if the multiprocessing logger is active if util._logger is not None: d["log_level"] = util._logger.getEffectiveLevel() if util._logger.handlers: h = util._logger.handlers[0] d["log_fmt"] = h.formatter._fmt # Tell the child how to communicate with the resource_tracker from .resource_tracker import _resource_tracker _resource_tracker.ensure_running() d["tracker_args"] = {"pid": _resource_tracker._pid} if sys.platform == "win32": d["tracker_args"]["fh"] = msvcrt.get_osfhandle(_resource_tracker._fd) else: d["tracker_args"]["fd"] = _resource_tracker._fd if sys.version_info >= (3, 8) and os.name == "posix": # joblib/loky#242: allow loky processes to retrieve the resource # tracker of their parent in case the child processes depickles # shared_memory objects, that are still tracked by multiprocessing's # resource_tracker by default. # XXX: this is a workaround that may be error prone: in the future, it # would be better to have loky subclass multiprocessing's shared_memory # to force registration of shared_memory segments via loky's # resource_tracker. from multiprocessing.resource_tracker import ( _resource_tracker as mp_resource_tracker, ) # multiprocessing's resource_tracker must be running before loky # process is created (othewise the child won't be able to use it if it # is created later on) mp_resource_tracker.ensure_running() d["mp_tracker_args"] = { "fd": mp_resource_tracker._fd, "pid": mp_resource_tracker._pid, } # Figure out whether to initialise main in the subprocess as a module # or through direct execution (or to leave it alone entirely) if init_main_module: main_module = sys.modules["__main__"] try: main_mod_name = getattr(main_module.__spec__, "name", None) except BaseException: main_mod_name = None if main_mod_name is not None: d["init_main_from_name"] = main_mod_name elif sys.platform != "win32" or (not WINEXE and not WINSERVICE): main_path = getattr(main_module, "__file__", None) if main_path is not None: if ( not os.path.isabs(main_path) and process.ORIGINAL_DIR is not None ): main_path = os.path.join(process.ORIGINAL_DIR, main_path) d["init_main_from_path"] = os.path.normpath(main_path) return d # # Prepare current process # old_main_modules = [] def prepare(data, parent_sentinel=None): """Try to get current process ready to unpickle process object.""" if "name" in data: process.current_process().name = data["name"] if "authkey" in data: process.current_process().authkey = data["authkey"] if "log_to_stderr" in data and data["log_to_stderr"]: util.log_to_stderr() if "log_level" in data: util.get_logger().setLevel(data["log_level"]) if "log_fmt" in data: import logging util.get_logger().handlers[0].setFormatter( logging.Formatter(data["log_fmt"]) ) if "sys_path" in data: sys.path = data["sys_path"] if "sys_argv" in data: sys.argv = data["sys_argv"] if "dir" in data: os.chdir(data["dir"]) if "orig_dir" in data: process.ORIGINAL_DIR = data["orig_dir"] if "mp_tracker_args" in data: from multiprocessing.resource_tracker import ( _resource_tracker as mp_resource_tracker, ) mp_resource_tracker._fd = data["mp_tracker_args"]["fd"] mp_resource_tracker._pid = data["mp_tracker_args"]["pid"] if "tracker_args" in data: from .resource_tracker import _resource_tracker _resource_tracker._pid = data["tracker_args"]["pid"] if sys.platform == "win32": handle = data["tracker_args"]["fh"] handle = duplicate(handle, source_process=parent_sentinel) _resource_tracker._fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) else: _resource_tracker._fd = data["tracker_args"]["fd"] if "init_main_from_name" in data: _fixup_main_from_name(data["init_main_from_name"]) elif "init_main_from_path" in data: _fixup_main_from_path(data["init_main_from_path"]) # Multiprocessing module helpers to fix up the main module in # spawned subprocesses def _fixup_main_from_name(mod_name): # __main__.py files for packages, directories, zip archives, etc, run # their "main only" code unconditionally, so we don't even try to # populate anything in __main__, nor do we make any changes to # __main__ attributes current_main = sys.modules["__main__"] if mod_name == "__main__" or mod_name.endswith(".__main__"): return # If this process was forked, __main__ may already be populated if getattr(current_main.__spec__, "name", None) == mod_name: return # Otherwise, __main__ may contain some non-main code where we need to # support unpickling it properly. We rerun it as __mp_main__ and make # the normal __main__ an alias to that old_main_modules.append(current_main) main_module = types.ModuleType("__mp_main__") main_content = runpy.run_module( mod_name, run_name="__mp_main__", alter_sys=True ) main_module.__dict__.update(main_content) sys.modules["__main__"] = sys.modules["__mp_main__"] = main_module def _fixup_main_from_path(main_path): # If this process was forked, __main__ may already be populated current_main = sys.modules["__main__"] # Unfortunately, the main ipython launch script historically had no # "if __name__ == '__main__'" guard, so we work around that # by treating it like a __main__.py file # See https://github.com/ipython/ipython/issues/4698 main_name = os.path.splitext(os.path.basename(main_path))[0] if main_name == "ipython": return # Otherwise, if __file__ already has the setting we expect, # there's nothing more to do if getattr(current_main, "__file__", None) == main_path: return # If the parent process has sent a path through rather than a module # name we assume it is an executable script that may contain # non-main code that needs to be executed old_main_modules.append(current_main) main_module = types.ModuleType("__mp_main__") main_content = runpy.run_path(main_path, run_name="__mp_main__") main_module.__dict__.update(main_content) sys.modules["__main__"] = sys.modules["__mp_main__"] = main_module joblib-1.3.2/joblib/externals/loky/backend/synchronize.py000066400000000000000000000267701446465525000235450ustar00rootroot00000000000000############################################################################### # Synchronization primitives based on our SemLock implementation # # author: Thomas Moreau and Olivier Grisel # # adapted from multiprocessing/synchronize.py (17/02/2017) # * Remove ctx argument for compatibility reason # * Registers a cleanup function with the loky resource_tracker to remove the # semaphore when the process dies instead. # # TODO: investigate which Python version is required to be able to use # multiprocessing.resource_tracker and therefore multiprocessing.synchronize # instead of a loky-specific fork. import os import sys import tempfile import threading import _multiprocessing from time import time as _time from multiprocessing import process, util from multiprocessing.context import assert_spawning from . import resource_tracker __all__ = [ "Lock", "RLock", "Semaphore", "BoundedSemaphore", "Condition", "Event", ] # Try to import the mp.synchronize module cleanly, if it fails # raise ImportError for platforms lacking a working sem_open implementation. # See issue 3770 try: from _multiprocessing import SemLock as _SemLock from _multiprocessing import sem_unlink except ImportError: raise ImportError( "This platform lacks a functioning sem_open" " implementation, therefore, the required" " synchronization primitives needed will not" " function, see issue 3770." ) # # Constants # RECURSIVE_MUTEX, SEMAPHORE = range(2) SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX # # Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock` # class SemLock: _rand = tempfile._RandomNameSequence() def __init__(self, kind, value, maxvalue, name=None): # unlink_now is only used on win32 or when we are using fork. unlink_now = False if name is None: # Try to find an unused name for the SemLock instance. for _ in range(100): try: self._semlock = _SemLock( kind, value, maxvalue, SemLock._make_name(), unlink_now ) except FileExistsError: # pragma: no cover pass else: break else: # pragma: no cover raise FileExistsError("cannot find name for semaphore") else: self._semlock = _SemLock(kind, value, maxvalue, name, unlink_now) self.name = name util.debug( f"created semlock with handle {self._semlock.handle} and name " f'"{self.name}"' ) self._make_methods() def _after_fork(obj): obj._semlock._after_fork() util.register_after_fork(self, _after_fork) # When the object is garbage collected or the # process shuts down we unlink the semaphore name resource_tracker.register(self._semlock.name, "semlock") util.Finalize( self, SemLock._cleanup, (self._semlock.name,), exitpriority=0 ) @staticmethod def _cleanup(name): try: sem_unlink(name) except FileNotFoundError: # Already unlinked, possibly by user code: ignore and make sure to # unregister the semaphore from the resource tracker. pass finally: resource_tracker.unregister(name, "semlock") def _make_methods(self): self.acquire = self._semlock.acquire self.release = self._semlock.release def __enter__(self): return self._semlock.acquire() def __exit__(self, *args): return self._semlock.release() def __getstate__(self): assert_spawning(self) sl = self._semlock h = sl.handle return (h, sl.kind, sl.maxvalue, sl.name) def __setstate__(self, state): self._semlock = _SemLock._rebuild(*state) util.debug( f'recreated blocker with handle {state[0]!r} and name "{state[3]}"' ) self._make_methods() @staticmethod def _make_name(): # OSX does not support long names for semaphores return f"/loky-{os.getpid()}-{next(SemLock._rand)}" # # Semaphore # class Semaphore(SemLock): def __init__(self, value=1): SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX) def get_value(self): if sys.platform == "darwin": raise NotImplementedError("OSX does not implement sem_getvalue") return self._semlock._get_value() def __repr__(self): try: value = self._semlock._get_value() except Exception: value = "unknown" return f"<{self.__class__.__name__}(value={value})>" # # Bounded semaphore # class BoundedSemaphore(Semaphore): def __init__(self, value=1): SemLock.__init__(self, SEMAPHORE, value, value) def __repr__(self): try: value = self._semlock._get_value() except Exception: value = "unknown" return ( f"<{self.__class__.__name__}(value={value}, " f"maxvalue={self._semlock.maxvalue})>" ) # # Non-recursive lock # class Lock(SemLock): def __init__(self): super().__init__(SEMAPHORE, 1, 1) def __repr__(self): try: if self._semlock._is_mine(): name = process.current_process().name if threading.current_thread().name != "MainThread": name = f"{name}|{threading.current_thread().name}" elif self._semlock._get_value() == 1: name = "None" elif self._semlock._count() > 0: name = "SomeOtherThread" else: name = "SomeOtherProcess" except Exception: name = "unknown" return f"<{self.__class__.__name__}(owner={name})>" # # Recursive lock # class RLock(SemLock): def __init__(self): super().__init__(RECURSIVE_MUTEX, 1, 1) def __repr__(self): try: if self._semlock._is_mine(): name = process.current_process().name if threading.current_thread().name != "MainThread": name = f"{name}|{threading.current_thread().name}" count = self._semlock._count() elif self._semlock._get_value() == 1: name, count = "None", 0 elif self._semlock._count() > 0: name, count = "SomeOtherThread", "nonzero" else: name, count = "SomeOtherProcess", "nonzero" except Exception: name, count = "unknown", "unknown" return f"<{self.__class__.__name__}({name}, {count})>" # # Condition variable # class Condition: def __init__(self, lock=None): self._lock = lock or RLock() self._sleeping_count = Semaphore(0) self._woken_count = Semaphore(0) self._wait_semaphore = Semaphore(0) self._make_methods() def __getstate__(self): assert_spawning(self) return ( self._lock, self._sleeping_count, self._woken_count, self._wait_semaphore, ) def __setstate__(self, state): ( self._lock, self._sleeping_count, self._woken_count, self._wait_semaphore, ) = state self._make_methods() def __enter__(self): return self._lock.__enter__() def __exit__(self, *args): return self._lock.__exit__(*args) def _make_methods(self): self.acquire = self._lock.acquire self.release = self._lock.release def __repr__(self): try: num_waiters = ( self._sleeping_count._semlock._get_value() - self._woken_count._semlock._get_value() ) except Exception: num_waiters = "unknown" return f"<{self.__class__.__name__}({self._lock}, {num_waiters})>" def wait(self, timeout=None): assert ( self._lock._semlock._is_mine() ), "must acquire() condition before using wait()" # indicate that this thread is going to sleep self._sleeping_count.release() # release lock count = self._lock._semlock._count() for _ in range(count): self._lock.release() try: # wait for notification or timeout return self._wait_semaphore.acquire(True, timeout) finally: # indicate that this thread has woken self._woken_count.release() # reacquire lock for _ in range(count): self._lock.acquire() def notify(self): assert self._lock._semlock._is_mine(), "lock is not owned" assert not self._wait_semaphore.acquire(False) # to take account of timeouts since last notify() we subtract # woken_count from sleeping_count and rezero woken_count while self._woken_count.acquire(False): res = self._sleeping_count.acquire(False) assert res if self._sleeping_count.acquire(False): # try grabbing a sleeper self._wait_semaphore.release() # wake up one sleeper self._woken_count.acquire() # wait for the sleeper to wake # rezero _wait_semaphore in case a timeout just happened self._wait_semaphore.acquire(False) def notify_all(self): assert self._lock._semlock._is_mine(), "lock is not owned" assert not self._wait_semaphore.acquire(False) # to take account of timeouts since last notify*() we subtract # woken_count from sleeping_count and rezero woken_count while self._woken_count.acquire(False): res = self._sleeping_count.acquire(False) assert res sleepers = 0 while self._sleeping_count.acquire(False): self._wait_semaphore.release() # wake up one sleeper sleepers += 1 if sleepers: for _ in range(sleepers): self._woken_count.acquire() # wait for a sleeper to wake # rezero wait_semaphore in case some timeouts just happened while self._wait_semaphore.acquire(False): pass def wait_for(self, predicate, timeout=None): result = predicate() if result: return result if timeout is not None: endtime = _time() + timeout else: endtime = None waittime = None while not result: if endtime is not None: waittime = endtime - _time() if waittime <= 0: break self.wait(waittime) result = predicate() return result # # Event # class Event: def __init__(self): self._cond = Condition(Lock()) self._flag = Semaphore(0) def is_set(self): with self._cond: if self._flag.acquire(False): self._flag.release() return True return False def set(self): with self._cond: self._flag.acquire(False) self._flag.release() self._cond.notify_all() def clear(self): with self._cond: self._flag.acquire(False) def wait(self, timeout=None): with self._cond: if self._flag.acquire(False): self._flag.release() else: self._cond.wait(timeout) if self._flag.acquire(False): self._flag.release() return True return False joblib-1.3.2/joblib/externals/loky/backend/utils.py000066400000000000000000000131751446465525000223250ustar00rootroot00000000000000import os import sys import time import errno import signal import warnings import subprocess import traceback try: import psutil except ImportError: psutil = None def kill_process_tree(process, use_psutil=True): """Terminate process and its descendants with SIGKILL""" if use_psutil and psutil is not None: _kill_process_tree_with_psutil(process) else: _kill_process_tree_without_psutil(process) def recursive_terminate(process, use_psutil=True): warnings.warn( "recursive_terminate is deprecated in loky 3.2, use kill_process_tree" "instead", DeprecationWarning, ) kill_process_tree(process, use_psutil=use_psutil) def _kill_process_tree_with_psutil(process): try: descendants = psutil.Process(process.pid).children(recursive=True) except psutil.NoSuchProcess: return # Kill the descendants in reverse order to avoid killing the parents before # the descendant in cases where there are more processes nested. for descendant in descendants[::-1]: try: descendant.kill() except psutil.NoSuchProcess: pass try: psutil.Process(process.pid).kill() except psutil.NoSuchProcess: pass process.join() def _kill_process_tree_without_psutil(process): """Terminate a process and its descendants.""" try: if sys.platform == "win32": _windows_taskkill_process_tree(process.pid) else: _posix_recursive_kill(process.pid) except Exception: # pragma: no cover details = traceback.format_exc() warnings.warn( "Failed to kill subprocesses on this platform. Please install" "psutil: https://github.com/giampaolo/psutil\n" f"Details:\n{details}" ) # In case we cannot introspect or kill the descendants, we fall back to # only killing the main process. # # Note: on Windows, process.kill() is an alias for process.terminate() # which in turns calls the Win32 API function TerminateProcess(). process.kill() process.join() def _windows_taskkill_process_tree(pid): # On windows, the taskkill function with option `/T` terminate a given # process pid and its children. try: subprocess.check_output( ["taskkill", "/F", "/T", "/PID", str(pid)], stderr=None ) except subprocess.CalledProcessError as e: # In Windows, taskkill returns 128, 255 for no process found. if e.returncode not in [128, 255]: # Let's raise to let the caller log the error details in a # warning and only kill the root process. raise # pragma: no cover def _kill(pid): # Not all systems (e.g. Windows) have a SIGKILL, but the C specification # mandates a SIGTERM signal. While Windows is handled specifically above, # let's try to be safe for other hypothetic platforms that only have # SIGTERM without SIGKILL. kill_signal = getattr(signal, "SIGKILL", signal.SIGTERM) try: os.kill(pid, kill_signal) except OSError as e: # if OSError is raised with [Errno 3] no such process, the process # is already terminated, else, raise the error and let the top # level function raise a warning and retry to kill the process. if e.errno != errno.ESRCH: raise # pragma: no cover def _posix_recursive_kill(pid): """Recursively kill the descendants of a process before killing it.""" try: children_pids = subprocess.check_output( ["pgrep", "-P", str(pid)], stderr=None, text=True ) except subprocess.CalledProcessError as e: # `ps` returns 1 when no child process has been found if e.returncode == 1: children_pids = "" else: raise # pragma: no cover # Decode the result, split the cpid and remove the trailing line for cpid in children_pids.splitlines(): cpid = int(cpid) _posix_recursive_kill(cpid) _kill(pid) def get_exitcodes_terminated_worker(processes): """Return a formatted string with the exitcodes of terminated workers. If necessary, wait (up to .25s) for the system to correctly set the exitcode of one terminated worker. """ patience = 5 # Catch the exitcode of the terminated workers. There should at least be # one. If not, wait a bit for the system to correctly set the exitcode of # the terminated worker. exitcodes = [ p.exitcode for p in list(processes.values()) if p.exitcode is not None ] while not exitcodes and patience > 0: patience -= 1 exitcodes = [ p.exitcode for p in list(processes.values()) if p.exitcode is not None ] time.sleep(0.05) return _format_exitcodes(exitcodes) def _format_exitcodes(exitcodes): """Format a list of exit code with names of the signals if possible""" str_exitcodes = [ f"{_get_exitcode_name(e)}({e})" for e in exitcodes if e is not None ] return "{" + ", ".join(str_exitcodes) + "}" def _get_exitcode_name(exitcode): if sys.platform == "win32": # The exitcode are unreliable on windows (see bpo-31863). # For this case, return UNKNOWN return "UNKNOWN" if exitcode < 0: try: import signal return signal.Signals(-exitcode).name except ValueError: return "UNKNOWN" elif exitcode != 255: # The exitcode are unreliable on forkserver were 255 is always returned # (see bpo-30589). For this case, return UNKNOWN return "EXIT" return "UNKNOWN" joblib-1.3.2/joblib/externals/loky/cloudpickle_wrapper.py000066400000000000000000000070301446465525000236250ustar00rootroot00000000000000import inspect from functools import partial from joblib.externals.cloudpickle import dumps, loads WRAP_CACHE = {} class CloudpickledObjectWrapper: def __init__(self, obj, keep_wrapper=False): self._obj = obj self._keep_wrapper = keep_wrapper def __reduce__(self): _pickled_object = dumps(self._obj) if not self._keep_wrapper: return loads, (_pickled_object,) return _reconstruct_wrapper, (_pickled_object, self._keep_wrapper) def __getattr__(self, attr): # Ensure that the wrapped object can be used seemlessly as the # previous object. if attr not in ["_obj", "_keep_wrapper"]: return getattr(self._obj, attr) return getattr(self, attr) # Make sure the wrapped object conserves the callable property class CallableObjectWrapper(CloudpickledObjectWrapper): def __call__(self, *args, **kwargs): return self._obj(*args, **kwargs) def _wrap_non_picklable_objects(obj, keep_wrapper): if callable(obj): return CallableObjectWrapper(obj, keep_wrapper=keep_wrapper) return CloudpickledObjectWrapper(obj, keep_wrapper=keep_wrapper) def _reconstruct_wrapper(_pickled_object, keep_wrapper): obj = loads(_pickled_object) return _wrap_non_picklable_objects(obj, keep_wrapper) def _wrap_objects_when_needed(obj): # Function to introspect an object and decide if it should be wrapped or # not. need_wrap = "__main__" in getattr(obj, "__module__", "") if isinstance(obj, partial): return partial( _wrap_objects_when_needed(obj.func), *[_wrap_objects_when_needed(a) for a in obj.args], **{ k: _wrap_objects_when_needed(v) for k, v in obj.keywords.items() } ) if callable(obj): # Need wrap if the object is a function defined in a local scope of # another function. func_code = getattr(obj, "__code__", "") need_wrap |= getattr(func_code, "co_flags", 0) & inspect.CO_NESTED # Need wrap if the obj is a lambda expression func_name = getattr(obj, "__name__", "") need_wrap |= "" in func_name if not need_wrap: return obj wrapped_obj = WRAP_CACHE.get(obj) if wrapped_obj is None: wrapped_obj = _wrap_non_picklable_objects(obj, keep_wrapper=False) WRAP_CACHE[obj] = wrapped_obj return wrapped_obj def wrap_non_picklable_objects(obj, keep_wrapper=True): """Wrapper for non-picklable object to use cloudpickle to serialize them. Note that this wrapper tends to slow down the serialization process as it is done with cloudpickle which is typically slower compared to pickle. The proper way to solve serialization issues is to avoid defining functions and objects in the main scripts and to implement __reduce__ functions for complex classes. """ # If obj is a class, create a CloudpickledClassWrapper which instantiates # the object internally and wrap it directly in a CloudpickledObjectWrapper if inspect.isclass(obj): class CloudpickledClassWrapper(CloudpickledObjectWrapper): def __init__(self, *args, **kwargs): self._obj = obj(*args, **kwargs) self._keep_wrapper = keep_wrapper CloudpickledClassWrapper.__name__ = obj.__name__ return CloudpickledClassWrapper # If obj is an instance of a class, just wrap it in a regular # CloudpickledObjectWrapper return _wrap_non_picklable_objects(obj, keep_wrapper=keep_wrapper) joblib-1.3.2/joblib/externals/loky/initializers.py000066400000000000000000000050071446465525000222770ustar00rootroot00000000000000import warnings def _viztracer_init(init_kwargs): """Initialize viztracer's profiler in worker processes""" from viztracer import VizTracer tracer = VizTracer(**init_kwargs) tracer.register_exit() tracer.start() def _make_viztracer_initializer_and_initargs(): try: import viztracer tracer = viztracer.get_tracer() if tracer is not None and getattr(tracer, "enable", False): # Profiler is active: introspect its configuration to # initialize the workers with the same configuration. return _viztracer_init, (tracer.init_kwargs,) except ImportError: # viztracer is not installed: nothing to do pass except Exception as e: # In case viztracer's API evolve, we do not want to crash loky but # we want to know about it to be able to update loky. warnings.warn(f"Unable to introspect viztracer state: {e}") return None, () class _ChainedInitializer: """Compound worker initializer This is meant to be used in conjunction with _chain_initializers to produce the necessary chained_args list to be passed to __call__. """ def __init__(self, initializers): self._initializers = initializers def __call__(self, *chained_args): for initializer, args in zip(self._initializers, chained_args): initializer(*args) def _chain_initializers(initializer_and_args): """Convenience helper to combine a sequence of initializers. If some initializers are None, they are filtered out. """ filtered_initializers = [] filtered_initargs = [] for initializer, initargs in initializer_and_args: if initializer is not None: filtered_initializers.append(initializer) filtered_initargs.append(initargs) if not filtered_initializers: return None, () elif len(filtered_initializers) == 1: return filtered_initializers[0], filtered_initargs[0] else: return _ChainedInitializer(filtered_initializers), filtered_initargs def _prepare_initializer(initializer, initargs): if initializer is not None and not callable(initializer): raise TypeError( f"initializer must be a callable, got: {initializer!r}" ) # Introspect runtime to determine if we need to propagate the viztracer # profiler information to the workers: return _chain_initializers( [ (initializer, initargs), _make_viztracer_initializer_and_initargs(), ] ) joblib-1.3.2/joblib/externals/loky/process_executor.py000066400000000000000000001435521446465525000231750ustar00rootroot00000000000000############################################################################### # Re-implementation of the ProcessPoolExecutor more robust to faults # # author: Thomas Moreau and Olivier Grisel # # adapted from concurrent/futures/process_pool_executor.py (17/02/2017) # * Add an extra management thread to detect executor_manager_thread failures, # * Improve the shutdown process to avoid deadlocks, # * Add timeout for workers, # * More robust pickling process. # # Copyright 2009 Brian Quinlan. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Implements ProcessPoolExecutor. The follow diagram and text describe the data-flow through the system: |======================= In-process =====================|== Out-of-process ==| +----------+ +----------+ +--------+ +-----------+ +---------+ | | => | Work Ids | | | | Call Q | | Process | | | +----------+ | | +-----------+ | Pool | | | | ... | | | | ... | +---------+ | | | 6 | => | | => | 5, call() | => | | | | | 7 | | | | ... | | | | Process | | ... | | Local | +-----------+ | Process | | Pool | +----------+ | Worker | | #1..n | | Executor | | Thread | | | | | +----------- + | | +-----------+ | | | | <=> | Work Items | <=> | | <= | Result Q | <= | | | | +------------+ | | +-----------+ | | | | | 6: call() | | | | ... | | | | | | future | +--------+ | 4, result | | | | | | ... | | 3, except | | | +----------+ +------------+ +-----------+ +---------+ Executor.submit() called: - creates a uniquely numbered _WorkItem and adds it to the "Work Items" dict - adds the id of the _WorkItem to the "Work Ids" queue Local worker thread: - reads work ids from the "Work Ids" queue and looks up the corresponding WorkItem from the "Work Items" dict: if the work item has been cancelled then it is simply removed from the dict, otherwise it is repackaged as a _CallItem and put in the "Call Q". New _CallItems are put in the "Call Q" until "Call Q" is full. NOTE: the size of the "Call Q" is kept small because calls placed in the "Call Q" can no longer be cancelled with Future.cancel(). - reads _ResultItems from "Result Q", updates the future stored in the "Work Items" dict and deletes the dict entry Process #1..n: - reads _CallItems from "Call Q", executes the calls, and puts the resulting _ResultItems in "Result Q" """ __author__ = "Thomas Moreau (thomas.moreau.2010@gmail.com)" import os import gc import sys import queue import struct import weakref import warnings import itertools import traceback import threading from time import time, sleep import multiprocessing as mp from functools import partial from pickle import PicklingError from concurrent.futures import Executor from concurrent.futures._base import LOGGER from concurrent.futures.process import BrokenProcessPool as _BPPException from multiprocessing.connection import wait from ._base import Future from .backend import get_context from .backend.context import cpu_count, _MAX_WINDOWS_WORKERS from .backend.queues import Queue, SimpleQueue from .backend.reduction import set_loky_pickler, get_loky_pickler_name from .backend.utils import kill_process_tree, get_exitcodes_terminated_worker from .initializers import _prepare_initializer # Mechanism to prevent infinite process spawning. When a worker of a # ProcessPoolExecutor nested in MAX_DEPTH Executor tries to create a new # Executor, a LokyRecursionError is raised MAX_DEPTH = int(os.environ.get("LOKY_MAX_DEPTH", 10)) _CURRENT_DEPTH = 0 # Minimum time interval between two consecutive memory leak protection checks. _MEMORY_LEAK_CHECK_DELAY = 1.0 # Number of bytes of memory usage allowed over the reference process size. _MAX_MEMORY_LEAK_SIZE = int(3e8) try: from psutil import Process _USE_PSUTIL = True def _get_memory_usage(pid, force_gc=False): if force_gc: gc.collect() mem_size = Process(pid).memory_info().rss mp.util.debug(f"psutil return memory size: {mem_size}") return mem_size except ImportError: _USE_PSUTIL = False class _ThreadWakeup: def __init__(self): self._closed = False self._reader, self._writer = mp.Pipe(duplex=False) def close(self): if not self._closed: self._closed = True self._writer.close() self._reader.close() def wakeup(self): if not self._closed: self._writer.send_bytes(b"") def clear(self): if not self._closed: while self._reader.poll(): self._reader.recv_bytes() class _ExecutorFlags: """necessary references to maintain executor states without preventing gc It permits to keep the information needed by executor_manager_thread and crash_detection_thread to maintain the pool without preventing the garbage collection of unreferenced executors. """ def __init__(self, shutdown_lock): self.shutdown = False self.broken = None self.kill_workers = False self.shutdown_lock = shutdown_lock def flag_as_shutting_down(self, kill_workers=None): with self.shutdown_lock: self.shutdown = True if kill_workers is not None: self.kill_workers = kill_workers def flag_as_broken(self, broken): with self.shutdown_lock: self.shutdown = True self.broken = broken # Prior to 3.9, executor_manager_thread is created as daemon thread. This means # that it is not joined automatically when the interpreter is shutting down. # To work around this problem, an exit handler is installed to tell the # thread to exit when the interpreter is shutting down and then waits until # it finishes. The thread needs to be daemonized because the atexit hooks are # called after all non daemonized threads are joined. # # Starting 3.9, there exists a specific atexit hook to be called before joining # the threads so the executor_manager_thread does not need to be daemonized # anymore. # # The atexit hooks are registered when starting the first ProcessPoolExecutor # to avoid import having an effect on the interpreter. _global_shutdown = False _global_shutdown_lock = threading.Lock() _threads_wakeups = weakref.WeakKeyDictionary() def _python_exit(): global _global_shutdown _global_shutdown = True # Materialize the list of items to avoid error due to iterating over # changing size dictionary. items = list(_threads_wakeups.items()) if len(items) > 0: mp.util.debug( "Interpreter shutting down. Waking up {len(items)}" f"executor_manager_thread:\n{items}" ) # Wake up the executor_manager_thread's so they can detect the interpreter # is shutting down and exit. for _, (shutdown_lock, thread_wakeup) in items: with shutdown_lock: thread_wakeup.wakeup() # Collect the executor_manager_thread's to make sure we exit cleanly. for thread, _ in items: # This locks is to prevent situations where an executor is gc'ed in one # thread while the atexit finalizer is running in another thread. This # can happen when joblib is used in pypy for instance. with _global_shutdown_lock: thread.join() # With the fork context, _thread_wakeups is propagated to children. # Clear it after fork to avoid some situation that can cause some # freeze when joining the workers. mp.util.register_after_fork(_threads_wakeups, lambda obj: obj.clear()) # Module variable to register the at_exit call process_pool_executor_at_exit = None # Controls how many more calls than processes will be queued in the call queue. # A smaller number will mean that processes spend more time idle waiting for # work while a larger number will make Future.cancel() succeed less frequently # (Futures in the call queue cannot be cancelled). EXTRA_QUEUED_CALLS = 1 class _RemoteTraceback(Exception): """Embed stringification of remote traceback in local traceback""" def __init__(self, tb=None): self.tb = f'\n"""\n{tb}"""' def __str__(self): return self.tb # Do not inherit from BaseException to mirror # concurrent.futures.process._ExceptionWithTraceback class _ExceptionWithTraceback: def __init__(self, exc): tb = getattr(exc, "__traceback__", None) if tb is None: _, _, tb = sys.exc_info() tb = traceback.format_exception(type(exc), exc, tb) tb = "".join(tb) self.exc = exc self.tb = tb def __reduce__(self): return _rebuild_exc, (self.exc, self.tb) def _rebuild_exc(exc, tb): exc.__cause__ = _RemoteTraceback(tb) return exc class _WorkItem: __slots__ = ["future", "fn", "args", "kwargs"] def __init__(self, future, fn, args, kwargs): self.future = future self.fn = fn self.args = args self.kwargs = kwargs class _ResultItem: def __init__(self, work_id, exception=None, result=None): self.work_id = work_id self.exception = exception self.result = result class _CallItem: def __init__(self, work_id, fn, args, kwargs): self.work_id = work_id self.fn = fn self.args = args self.kwargs = kwargs # Store the current loky_pickler so it is correctly set in the worker self.loky_pickler = get_loky_pickler_name() def __call__(self): set_loky_pickler(self.loky_pickler) return self.fn(*self.args, **self.kwargs) def __repr__(self): return ( f"CallItem({self.work_id}, {self.fn}, {self.args}, {self.kwargs})" ) class _SafeQueue(Queue): """Safe Queue set exception to the future object linked to a job""" def __init__( self, max_size=0, ctx=None, pending_work_items=None, running_work_items=None, thread_wakeup=None, reducers=None, ): self.thread_wakeup = thread_wakeup self.pending_work_items = pending_work_items self.running_work_items = running_work_items super().__init__(max_size, reducers=reducers, ctx=ctx) def _on_queue_feeder_error(self, e, obj): if isinstance(obj, _CallItem): # format traceback only works on python3 if isinstance(e, struct.error): raised_error = RuntimeError( "The task could not be sent to the workers as it is too " "large for `send_bytes`." ) else: raised_error = PicklingError( "Could not pickle the task to send it to the workers." ) tb = traceback.format_exception( type(e), e, getattr(e, "__traceback__", None) ) raised_error.__cause__ = _RemoteTraceback("".join(tb)) work_item = self.pending_work_items.pop(obj.work_id, None) self.running_work_items.remove(obj.work_id) # work_item can be None if another process terminated. In this # case, the executor_manager_thread fails all work_items with # BrokenProcessPool if work_item is not None: work_item.future.set_exception(raised_error) del work_item self.thread_wakeup.wakeup() else: super()._on_queue_feeder_error(e, obj) def _get_chunks(chunksize, *iterables): """Iterates over zip()ed iterables in chunks.""" it = zip(*iterables) while True: chunk = tuple(itertools.islice(it, chunksize)) if not chunk: return yield chunk def _process_chunk(fn, chunk): """Processes a chunk of an iterable passed to map. Runs the function passed to map() on a chunk of the iterable passed to map. This function is run in a separate process. """ return [fn(*args) for args in chunk] def _sendback_result(result_queue, work_id, result=None, exception=None): """Safely send back the given result or exception""" try: result_queue.put( _ResultItem(work_id, result=result, exception=exception) ) except BaseException as e: exc = _ExceptionWithTraceback(e) result_queue.put(_ResultItem(work_id, exception=exc)) def _process_worker( call_queue, result_queue, initializer, initargs, processes_management_lock, timeout, worker_exit_lock, current_depth, ): """Evaluates calls from call_queue and places the results in result_queue. This worker is run in a separate process. Args: call_queue: A ctx.Queue of _CallItems that will be read and evaluated by the worker. result_queue: A ctx.Queue of _ResultItems that will written to by the worker. initializer: A callable initializer, or None initargs: A tuple of args for the initializer processes_management_lock: A ctx.Lock avoiding worker timeout while some workers are being spawned. timeout: maximum time to wait for a new item in the call_queue. If that time is expired, the worker will shutdown. worker_exit_lock: Lock to avoid flagging the executor as broken on workers timeout. current_depth: Nested parallelism level, to avoid infinite spawning. """ if initializer is not None: try: initializer(*initargs) except BaseException: LOGGER.critical("Exception in initializer:", exc_info=True) # The parent will notice that the process stopped and # mark the pool broken return # set the global _CURRENT_DEPTH mechanism to limit recursive call global _CURRENT_DEPTH _CURRENT_DEPTH = current_depth _process_reference_size = None _last_memory_leak_check = None pid = os.getpid() mp.util.debug(f"Worker started with timeout={timeout}") while True: try: call_item = call_queue.get(block=True, timeout=timeout) if call_item is None: mp.util.info("Shutting down worker on sentinel") except queue.Empty: mp.util.info(f"Shutting down worker after timeout {timeout:0.3f}s") if processes_management_lock.acquire(block=False): processes_management_lock.release() call_item = None else: mp.util.info("Could not acquire processes_management_lock") continue except BaseException: previous_tb = traceback.format_exc() try: result_queue.put(_RemoteTraceback(previous_tb)) except BaseException: # If we cannot format correctly the exception, at least print # the traceback. print(previous_tb) mp.util.debug("Exiting with code 1") sys.exit(1) if call_item is None: # Notify queue management thread about worker shutdown result_queue.put(pid) is_clean = worker_exit_lock.acquire(True, timeout=30) # Early notify any loky executor running in this worker process # (nested parallelism) that this process is about to shutdown to # avoid a deadlock waiting undifinitely for the worker to finish. _python_exit() if is_clean: mp.util.debug("Exited cleanly") else: mp.util.info("Main process did not release worker_exit") return try: r = call_item() except BaseException as e: exc = _ExceptionWithTraceback(e) result_queue.put(_ResultItem(call_item.work_id, exception=exc)) else: _sendback_result(result_queue, call_item.work_id, result=r) del r # Free the resource as soon as possible, to avoid holding onto # open files or shared memory that is not needed anymore del call_item if _USE_PSUTIL: if _process_reference_size is None: # Make reference measurement after the first call _process_reference_size = _get_memory_usage(pid, force_gc=True) _last_memory_leak_check = time() continue if time() - _last_memory_leak_check > _MEMORY_LEAK_CHECK_DELAY: mem_usage = _get_memory_usage(pid) _last_memory_leak_check = time() if mem_usage - _process_reference_size < _MAX_MEMORY_LEAK_SIZE: # Memory usage stays within bounds: everything is fine. continue # Check again memory usage; this time take the measurement # after a forced garbage collection to break any reference # cycles. mem_usage = _get_memory_usage(pid, force_gc=True) _last_memory_leak_check = time() if mem_usage - _process_reference_size < _MAX_MEMORY_LEAK_SIZE: # The GC managed to free the memory: everything is fine. continue # The process is leaking memory: let the master process # know that we need to start a new worker. mp.util.info("Memory leak detected: shutting down worker") result_queue.put(pid) with worker_exit_lock: mp.util.debug("Exit due to memory leak") return else: # if psutil is not installed, trigger gc.collect events # regularly to limit potential memory leaks due to reference cycles if _last_memory_leak_check is None or ( time() - _last_memory_leak_check > _MEMORY_LEAK_CHECK_DELAY ): gc.collect() _last_memory_leak_check = time() class _ExecutorManagerThread(threading.Thread): """Manages the communication between this process and the worker processes. The manager is run in a local thread. Args: executor: A reference to the ProcessPoolExecutor that owns this thread. A weakref will be own by the manager as well as references to internal objects used to introspect the state of the executor. """ def __init__(self, executor): # Store references to necessary internals of the executor. # A _ThreadWakeup to allow waking up the executor_manager_thread from # the main Thread and avoid deadlocks caused by permanently # locked queues. self.thread_wakeup = executor._executor_manager_thread_wakeup self.shutdown_lock = executor._shutdown_lock # A weakref.ref to the ProcessPoolExecutor that owns this thread. Used # to determine if the ProcessPoolExecutor has been garbage collected # and that the manager can exit. # When the executor gets garbage collected, the weakref callback # will wake up the queue management thread so that it can terminate # if there is no pending work item. def weakref_cb( _, thread_wakeup=self.thread_wakeup, shutdown_lock=self.shutdown_lock, ): if mp is not None: # At this point, the multiprocessing module can already be # garbage collected. We only log debug info when still # possible. mp.util.debug( "Executor collected: triggering callback for" " QueueManager wakeup" ) with shutdown_lock: thread_wakeup.wakeup() self.executor_reference = weakref.ref(executor, weakref_cb) # The flags of the executor self.executor_flags = executor._flags # A list of the ctx.Process instances used as workers. self.processes = executor._processes # A ctx.Queue that will be filled with _CallItems derived from # _WorkItems for processing by the process workers. self.call_queue = executor._call_queue # A ctx.SimpleQueue of _ResultItems generated by the process workers. self.result_queue = executor._result_queue # A queue.Queue of work ids e.g. Queue([5, 6, ...]). self.work_ids_queue = executor._work_ids # A dict mapping work ids to _WorkItems e.g. # {5: <_WorkItem...>, 6: <_WorkItem...>, ...} self.pending_work_items = executor._pending_work_items # A list of the work_ids that are currently running self.running_work_items = executor._running_work_items # A lock to avoid concurrent shutdown of workers on timeout and spawn # of new processes or shut down self.processes_management_lock = executor._processes_management_lock super().__init__(name="ExecutorManagerThread") if sys.version_info < (3, 9): self.daemon = True def run(self): # Main loop for the executor manager thread. while True: self.add_call_item_to_queue() result_item, is_broken, bpe = self.wait_result_broken_or_wakeup() if is_broken: self.terminate_broken(bpe) return if result_item is not None: self.process_result_item(result_item) # Delete reference to result_item to avoid keeping references # while waiting on new results. del result_item if self.is_shutting_down(): self.flag_executor_shutting_down() # Since no new work items can be added, it is safe to shutdown # this thread if there are no pending work items. if not self.pending_work_items: self.join_executor_internals() return def add_call_item_to_queue(self): # Fills call_queue with _WorkItems from pending_work_items. # This function never blocks. while True: if self.call_queue.full(): return try: work_id = self.work_ids_queue.get(block=False) except queue.Empty: return else: work_item = self.pending_work_items[work_id] if work_item.future.set_running_or_notify_cancel(): self.running_work_items += [work_id] self.call_queue.put( _CallItem( work_id, work_item.fn, work_item.args, work_item.kwargs, ), block=True, ) else: del self.pending_work_items[work_id] continue def wait_result_broken_or_wakeup(self): # Wait for a result to be ready in the result_queue while checking # that all worker processes are still running, or for a wake up # signal send. The wake up signals come either from new tasks being # submitted, from the executor being shutdown/gc-ed, or from the # shutdown of the python interpreter. result_reader = self.result_queue._reader wakeup_reader = self.thread_wakeup._reader readers = [result_reader, wakeup_reader] worker_sentinels = [p.sentinel for p in list(self.processes.values())] ready = wait(readers + worker_sentinels) bpe = None is_broken = True result_item = None if result_reader in ready: try: result_item = result_reader.recv() if isinstance(result_item, _RemoteTraceback): bpe = BrokenProcessPool( "A task has failed to un-serialize. Please ensure that" " the arguments of the function are all picklable." ) bpe.__cause__ = result_item else: is_broken = False except BaseException as e: bpe = BrokenProcessPool( "A result has failed to un-serialize. Please ensure that " "the objects returned by the function are always " "picklable." ) tb = traceback.format_exception( type(e), e, getattr(e, "__traceback__", None) ) bpe.__cause__ = _RemoteTraceback("".join(tb)) elif wakeup_reader in ready: # This is simply a wake-up event that might either trigger putting # more tasks in the queue or trigger the clean up of resources. is_broken = False else: # A worker has terminated and we don't know why, set the state of # the executor as broken exit_codes = "" if sys.platform != "win32": # In Windows, introspecting terminated workers exitcodes seems # unstable, therefore they are not appended in the exception # message. exit_codes = ( "\nThe exit codes of the workers are " f"{get_exitcodes_terminated_worker(self.processes)}" ) mp.util.debug( "A worker unexpectedly terminated. Workers that " "might have caused the breakage: " + str( { p.name: p.exitcode for p in list(self.processes.values()) if p is not None and p.sentinel in ready } ) ) bpe = TerminatedWorkerError( "A worker process managed by the executor was unexpectedly " "terminated. This could be caused by a segmentation fault " "while calling the function or by an excessive memory usage " "causing the Operating System to kill the worker.\n" f"{exit_codes}" ) self.thread_wakeup.clear() return result_item, is_broken, bpe def process_result_item(self, result_item): # Process the received a result_item. This can be either the PID of a # worker that exited gracefully or a _ResultItem if isinstance(result_item, int): # Clean shutdown of a worker using its PID, either on request # by the executor.shutdown method or by the timeout of the worker # itself: we should not mark the executor as broken. with self.processes_management_lock: p = self.processes.pop(result_item, None) # p can be None if the executor is concurrently shutting down. if p is not None: p._worker_exit_lock.release() mp.util.debug( f"joining {p.name} when processing {p.pid} as result_item" ) p.join() del p # Make sure the executor have the right number of worker, even if a # worker timeout while some jobs were submitted. If some work is # pending or there is less processes than running items, we need to # start a new Process and raise a warning. n_pending = len(self.pending_work_items) n_running = len(self.running_work_items) if n_pending - n_running > 0 or n_running > len(self.processes): executor = self.executor_reference() if ( executor is not None and len(self.processes) < executor._max_workers ): warnings.warn( "A worker stopped while some jobs were given to the " "executor. This can be caused by a too short worker " "timeout or by a memory leak.", UserWarning, ) with executor._processes_management_lock: executor._adjust_process_count() executor = None else: # Received a _ResultItem so mark the future as completed. work_item = self.pending_work_items.pop(result_item.work_id, None) # work_item can be None if another process terminated (see above) if work_item is not None: if result_item.exception: work_item.future.set_exception(result_item.exception) else: work_item.future.set_result(result_item.result) self.running_work_items.remove(result_item.work_id) def is_shutting_down(self): # Check whether we should start shutting down the executor. executor = self.executor_reference() # No more work items can be added if: # - The interpreter is shutting down OR # - The executor that owns this thread is not broken AND # * The executor that owns this worker has been collected OR # * The executor that owns this worker has been shutdown. # If the executor is broken, it should be detected in the next loop. return _global_shutdown or ( (executor is None or self.executor_flags.shutdown) and not self.executor_flags.broken ) def terminate_broken(self, bpe): # Terminate the executor because it is in a broken state. The bpe # argument can be used to display more information on the error that # lead the executor into becoming broken. # Mark the process pool broken so that submits fail right now. self.executor_flags.flag_as_broken(bpe) # Mark pending tasks as failed. for work_item in self.pending_work_items.values(): work_item.future.set_exception(bpe) # Delete references to object. See issue16284 del work_item self.pending_work_items.clear() # Terminate remaining workers forcibly: the queues or their # locks may be in a dirty state and block forever. self.kill_workers(reason="broken executor") # clean up resources self.join_executor_internals() def flag_executor_shutting_down(self): # Flag the executor as shutting down and cancel remaining tasks if # requested as early as possible if it is not gc-ed yet. self.executor_flags.flag_as_shutting_down() # Cancel pending work items if requested. if self.executor_flags.kill_workers: while self.pending_work_items: _, work_item = self.pending_work_items.popitem() work_item.future.set_exception( ShutdownExecutorError( "The Executor was shutdown with `kill_workers=True` " "before this job could complete." ) ) del work_item # Kill the remaining worker forcibly to no waste time joining them self.kill_workers(reason="executor shutting down") def kill_workers(self, reason=""): # Terminate the remaining workers using SIGKILL. This function also # terminates descendant workers of the children in case there is some # nested parallelism. while self.processes: _, p = self.processes.popitem() mp.util.debug(f"terminate process {p.name}, reason: {reason}") try: kill_process_tree(p) except ProcessLookupError: # pragma: no cover pass def shutdown_workers(self): # shutdown all workers in self.processes # Create a list to avoid RuntimeError due to concurrent modification of # processes. nb_children_alive is thus an upper bound. Also release the # processes' _worker_exit_lock to accelerate the shutdown procedure, as # there is no need for hand-shake here. with self.processes_management_lock: n_children_to_stop = 0 for p in list(self.processes.values()): mp.util.debug(f"releasing worker exit lock on {p.name}") p._worker_exit_lock.release() n_children_to_stop += 1 mp.util.debug(f"found {n_children_to_stop} processes to stop") # Send the right number of sentinels, to make sure all children are # properly terminated. Do it with a mechanism that avoid hanging on # Full queue when all workers have already been shutdown. n_sentinels_sent = 0 cooldown_time = 0.001 while ( n_sentinels_sent < n_children_to_stop and self.get_n_children_alive() > 0 ): for _ in range(n_children_to_stop - n_sentinels_sent): try: self.call_queue.put_nowait(None) n_sentinels_sent += 1 except queue.Full as e: if cooldown_time > 5.0: mp.util.info( "failed to send all sentinels and exit with error." f"\ncall_queue size={self.call_queue._maxsize}; " f" full is {self.call_queue.full()}; " ) raise e mp.util.info( "full call_queue prevented to send all sentinels at " "once, waiting..." ) sleep(cooldown_time) cooldown_time *= 1.2 break mp.util.debug(f"sent {n_sentinels_sent} sentinels to the call queue") def join_executor_internals(self): self.shutdown_workers() # Release the queue's resources as soon as possible. Flag the feeder # thread for clean exit to avoid having the crash detection thread flag # the Executor as broken during the shutdown. This is safe as either: # * We don't need to communicate with the workers anymore # * There is nothing left in the Queue buffer except None sentinels mp.util.debug("closing call_queue") self.call_queue.close() self.call_queue.join_thread() # Closing result_queue mp.util.debug("closing result_queue") self.result_queue.close() mp.util.debug("closing thread_wakeup") with self.shutdown_lock: self.thread_wakeup.close() # If .join() is not called on the created processes then # some ctx.Queue methods may deadlock on macOS. with self.processes_management_lock: mp.util.debug(f"joining {len(self.processes)} processes") n_joined_processes = 0 while True: try: pid, p = self.processes.popitem() mp.util.debug(f"joining process {p.name} with pid {pid}") p.join() n_joined_processes += 1 except KeyError: break mp.util.debug( "executor management thread clean shutdown of " f"{n_joined_processes} workers" ) def get_n_children_alive(self): # This is an upper bound on the number of children alive. with self.processes_management_lock: return sum(p.is_alive() for p in list(self.processes.values())) _system_limits_checked = False _system_limited = None def _check_system_limits(): global _system_limits_checked, _system_limited if _system_limits_checked and _system_limited: raise NotImplementedError(_system_limited) _system_limits_checked = True try: nsems_max = os.sysconf("SC_SEM_NSEMS_MAX") except (AttributeError, ValueError): # sysconf not available or setting not available return if nsems_max == -1: # undetermined limit, assume that limit is determined # by available memory only return if nsems_max >= 256: # minimum number of semaphores available # according to POSIX return _system_limited = ( f"system provides too few semaphores ({nsems_max} available, " "256 necessary)" ) raise NotImplementedError(_system_limited) def _chain_from_iterable_of_lists(iterable): """ Specialized implementation of itertools.chain.from_iterable. Each item in *iterable* should be a list. This function is careful not to keep references to yielded objects. """ for element in iterable: element.reverse() while element: yield element.pop() def _check_max_depth(context): # Limit the maxmal recursion level global _CURRENT_DEPTH if context.get_start_method() == "fork" and _CURRENT_DEPTH > 0: raise LokyRecursionError( "Could not spawn extra nested processes at depth superior to " "MAX_DEPTH=1. It is not possible to increase this limit when " "using the 'fork' start method." ) if 0 < MAX_DEPTH and _CURRENT_DEPTH + 1 > MAX_DEPTH: raise LokyRecursionError( "Could not spawn extra nested processes at depth superior to " f"MAX_DEPTH={MAX_DEPTH}. If this is intendend, you can change " "this limit with the LOKY_MAX_DEPTH environment variable." ) class LokyRecursionError(RuntimeError): """A process tries to spawn too many levels of nested processes.""" class BrokenProcessPool(_BPPException): """ Raised when the executor is broken while a future was in the running state. The cause can an error raised when unpickling the task in the worker process or when unpickling the result value in the parent process. It can also be caused by a worker process being terminated unexpectedly. """ class TerminatedWorkerError(BrokenProcessPool): """ Raised when a process in a ProcessPoolExecutor terminated abruptly while a future was in the running state. """ # Alias for backward compat (for code written for loky 1.1.4 and earlier). Do # not use in new code. BrokenExecutor = BrokenProcessPool class ShutdownExecutorError(RuntimeError): """ Raised when a ProcessPoolExecutor is shutdown while a future was in the running or pending state. """ class ProcessPoolExecutor(Executor): _at_exit = None def __init__( self, max_workers=None, job_reducers=None, result_reducers=None, timeout=None, context=None, initializer=None, initargs=(), env=None, ): """Initializes a new ProcessPoolExecutor instance. Args: max_workers: int, optional (default: cpu_count()) The maximum number of processes that can be used to execute the given calls. If None or not given then as many worker processes will be created as the number of CPUs the current process can use. job_reducers, result_reducers: dict(type: reducer_func) Custom reducer for pickling the jobs and the results from the Executor. If only `job_reducers` is provided, `result_reducer` will use the same reducers timeout: int, optional (default: None) Idle workers exit after timeout seconds. If a new job is submitted after the timeout, the executor will start enough new Python processes to make sure the pool of workers is full. context: A multiprocessing context to launch the workers. This object should provide SimpleQueue, Queue and Process. initializer: An callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. env: A dict of environment variable to overwrite in the child process. The environment variables are set before any module is loaded. Note that this only works with the loky context. """ _check_system_limits() if max_workers is None: self._max_workers = cpu_count() else: if max_workers <= 0: raise ValueError("max_workers must be greater than 0") self._max_workers = max_workers if ( sys.platform == "win32" and self._max_workers > _MAX_WINDOWS_WORKERS ): warnings.warn( f"On Windows, max_workers cannot exceed {_MAX_WINDOWS_WORKERS} " "due to limitations of the operating system." ) self._max_workers = _MAX_WINDOWS_WORKERS if context is None: context = get_context() self._context = context self._env = env self._initializer, self._initargs = _prepare_initializer( initializer, initargs ) _check_max_depth(self._context) if result_reducers is None: result_reducers = job_reducers # Timeout self._timeout = timeout # Management thread self._executor_manager_thread = None # Map of pids to processes self._processes = {} # Internal variables of the ProcessPoolExecutor self._processes = {} self._queue_count = 0 self._pending_work_items = {} self._running_work_items = [] self._work_ids = queue.Queue() self._processes_management_lock = self._context.Lock() self._executor_manager_thread = None self._shutdown_lock = threading.Lock() # _ThreadWakeup is a communication channel used to interrupt the wait # of the main loop of executor_manager_thread from another thread (e.g. # when calling executor.submit or executor.shutdown). We do not use the # _result_queue to send wakeup signals to the executor_manager_thread # as it could result in a deadlock if a worker process dies with the # _result_queue write lock still acquired. # # _shutdown_lock must be locked to access _ThreadWakeup.wakeup. self._executor_manager_thread_wakeup = _ThreadWakeup() # Flag to hold the state of the Executor. This permits to introspect # the Executor state even once it has been garbage collected. self._flags = _ExecutorFlags(self._shutdown_lock) # Finally setup the queues for interprocess communication self._setup_queues(job_reducers, result_reducers) mp.util.debug("ProcessPoolExecutor is setup") def _setup_queues(self, job_reducers, result_reducers, queue_size=None): # Make the call queue slightly larger than the number of processes to # prevent the worker processes from idling. But don't make it too big # because futures in the call queue cannot be cancelled. if queue_size is None: queue_size = 2 * self._max_workers + EXTRA_QUEUED_CALLS self._call_queue = _SafeQueue( max_size=queue_size, pending_work_items=self._pending_work_items, running_work_items=self._running_work_items, thread_wakeup=self._executor_manager_thread_wakeup, reducers=job_reducers, ctx=self._context, ) # Killed worker processes can produce spurious "broken pipe" # tracebacks in the queue's own worker thread. But we detect killed # processes anyway, so silence the tracebacks. self._call_queue._ignore_epipe = True self._result_queue = SimpleQueue( reducers=result_reducers, ctx=self._context ) def _start_executor_manager_thread(self): if self._executor_manager_thread is None: mp.util.debug("_start_executor_manager_thread called") # Start the processes so that their sentinels are known. self._executor_manager_thread = _ExecutorManagerThread(self) self._executor_manager_thread.start() # register this executor in a mechanism that ensures it will wakeup # when the interpreter is exiting. _threads_wakeups[self._executor_manager_thread] = ( self._shutdown_lock, self._executor_manager_thread_wakeup, ) global process_pool_executor_at_exit if process_pool_executor_at_exit is None: # Ensure that the _python_exit function will be called before # the multiprocessing.Queue._close finalizers which have an # exitpriority of 10. if sys.version_info < (3, 9): process_pool_executor_at_exit = mp.util.Finalize( None, _python_exit, exitpriority=20 ) else: process_pool_executor_at_exit = threading._register_atexit( _python_exit ) def _adjust_process_count(self): while len(self._processes) < self._max_workers: worker_exit_lock = self._context.BoundedSemaphore(1) args = ( self._call_queue, self._result_queue, self._initializer, self._initargs, self._processes_management_lock, self._timeout, worker_exit_lock, _CURRENT_DEPTH + 1, ) worker_exit_lock.acquire() try: # Try to spawn the process with some environment variable to # overwrite but it only works with the loky context for now. p = self._context.Process( target=_process_worker, args=args, env=self._env ) except TypeError: p = self._context.Process(target=_process_worker, args=args) p._worker_exit_lock = worker_exit_lock p.start() self._processes[p.pid] = p mp.util.debug( f"Adjusted process count to {self._max_workers}: " f"{[(p.name, pid) for pid, p in self._processes.items()]}" ) def _ensure_executor_running(self): """ensures all workers and management thread are running""" with self._processes_management_lock: if len(self._processes) != self._max_workers: self._adjust_process_count() self._start_executor_manager_thread() def submit(self, fn, *args, **kwargs): with self._flags.shutdown_lock: if self._flags.broken is not None: raise self._flags.broken if self._flags.shutdown: raise ShutdownExecutorError( "cannot schedule new futures after shutdown" ) # Cannot submit a new calls once the interpreter is shutting down. # This check avoids spawning new processes at exit. if _global_shutdown: raise RuntimeError( "cannot schedule new futures after " "interpreter shutdown" ) f = Future() w = _WorkItem(f, fn, args, kwargs) self._pending_work_items[self._queue_count] = w self._work_ids.put(self._queue_count) self._queue_count += 1 # Wake up queue management thread self._executor_manager_thread_wakeup.wakeup() self._ensure_executor_running() return f submit.__doc__ = Executor.submit.__doc__ def map(self, fn, *iterables, **kwargs): """Returns an iterator equivalent to map(fn, iter). Args: fn: A callable that will take as many arguments as there are passed iterables. timeout: The maximum number of seconds to wait. If None, then there is no limit on the wait time. chunksize: If greater than one, the iterables will be chopped into chunks of size chunksize and submitted to the process pool. If set to one, the items in the list will be sent one at a time. Returns: An iterator equivalent to: map(func, *iterables) but the calls may be evaluated out-of-order. Raises: TimeoutError: If the entire result iterator could not be generated before the given timeout. Exception: If fn(*args) raises for any values. """ timeout = kwargs.get("timeout", None) chunksize = kwargs.get("chunksize", 1) if chunksize < 1: raise ValueError("chunksize must be >= 1.") results = super().map( partial(_process_chunk, fn), _get_chunks(chunksize, *iterables), timeout=timeout, ) return _chain_from_iterable_of_lists(results) def shutdown(self, wait=True, kill_workers=False): mp.util.debug(f"shutting down executor {self}") self._flags.flag_as_shutting_down(kill_workers) executor_manager_thread = self._executor_manager_thread executor_manager_thread_wakeup = self._executor_manager_thread_wakeup if executor_manager_thread_wakeup is not None: # Wake up queue management thread with self._shutdown_lock: self._executor_manager_thread_wakeup.wakeup() if executor_manager_thread is not None and wait: # This locks avoids concurrent join if the interpreter # is shutting down. with _global_shutdown_lock: executor_manager_thread.join() _threads_wakeups.pop(executor_manager_thread, None) # To reduce the risk of opening too many files, remove references to # objects that use file descriptors. self._executor_manager_thread = None self._executor_manager_thread_wakeup = None self._call_queue = None self._result_queue = None self._processes_management_lock = None shutdown.__doc__ = Executor.shutdown.__doc__ joblib-1.3.2/joblib/externals/loky/reusable_executor.py000066400000000000000000000241011446465525000233050ustar00rootroot00000000000000############################################################################### # Reusable ProcessPoolExecutor # # author: Thomas Moreau and Olivier Grisel # import time import warnings import threading import multiprocessing as mp from .process_executor import ProcessPoolExecutor, EXTRA_QUEUED_CALLS from .backend.context import cpu_count from .backend import get_context __all__ = ["get_reusable_executor"] # Singleton executor and id management _executor_lock = threading.RLock() _next_executor_id = 0 _executor = None _executor_kwargs = None def _get_next_executor_id(): """Ensure that each successive executor instance has a unique, monotonic id. The purpose of this monotonic id is to help debug and test automated instance creation. """ global _next_executor_id with _executor_lock: executor_id = _next_executor_id _next_executor_id += 1 return executor_id def get_reusable_executor( max_workers=None, context=None, timeout=10, kill_workers=False, reuse="auto", job_reducers=None, result_reducers=None, initializer=None, initargs=(), env=None, ): """Return the current ReusableExectutor instance. Start a new instance if it has not been started already or if the previous instance was left in a broken state. If the previous instance does not have the requested number of workers, the executor is dynamically resized to adjust the number of workers prior to returning. Reusing a singleton instance spares the overhead of starting new worker processes and importing common python packages each time. ``max_workers`` controls the maximum number of tasks that can be running in parallel in worker processes. By default this is set to the number of CPUs on the host. Setting ``timeout`` (in seconds) makes idle workers automatically shutdown so as to release system resources. New workers are respawn upon submission of new tasks so that ``max_workers`` are available to accept the newly submitted tasks. Setting ``timeout`` to around 100 times the time required to spawn new processes and import packages in them (on the order of 100ms) ensures that the overhead of spawning workers is negligible. Setting ``kill_workers=True`` makes it possible to forcibly interrupt previously spawned jobs to get a new instance of the reusable executor with new constructor argument values. The ``job_reducers`` and ``result_reducers`` are used to customize the pickling of tasks and results send to the executor. When provided, the ``initializer`` is run first in newly spawned processes with argument ``initargs``. The environment variable in the child process are a copy of the values in the main process. One can provide a dict ``{ENV: VAL}`` where ``ENV`` and ``VAL`` are string literals to overwrite the environment variable ``ENV`` in the child processes to value ``VAL``. The environment variables are set in the children before any module is loaded. This only works with the ``loky`` context. """ _executor, _ = _ReusablePoolExecutor.get_reusable_executor( max_workers=max_workers, context=context, timeout=timeout, kill_workers=kill_workers, reuse=reuse, job_reducers=job_reducers, result_reducers=result_reducers, initializer=initializer, initargs=initargs, env=env, ) return _executor class _ReusablePoolExecutor(ProcessPoolExecutor): def __init__( self, submit_resize_lock, max_workers=None, context=None, timeout=None, executor_id=0, job_reducers=None, result_reducers=None, initializer=None, initargs=(), env=None, ): super().__init__( max_workers=max_workers, context=context, timeout=timeout, job_reducers=job_reducers, result_reducers=result_reducers, initializer=initializer, initargs=initargs, env=env, ) self.executor_id = executor_id self._submit_resize_lock = submit_resize_lock @classmethod def get_reusable_executor( cls, max_workers=None, context=None, timeout=10, kill_workers=False, reuse="auto", job_reducers=None, result_reducers=None, initializer=None, initargs=(), env=None, ): with _executor_lock: global _executor, _executor_kwargs executor = _executor if max_workers is None: if reuse is True and executor is not None: max_workers = executor._max_workers else: max_workers = cpu_count() elif max_workers <= 0: raise ValueError( f"max_workers must be greater than 0, got {max_workers}." ) if isinstance(context, str): context = get_context(context) if context is not None and context.get_start_method() == "fork": raise ValueError( "Cannot use reusable executor with the 'fork' context" ) kwargs = dict( context=context, timeout=timeout, job_reducers=job_reducers, result_reducers=result_reducers, initializer=initializer, initargs=initargs, env=env, ) if executor is None: is_reused = False mp.util.debug( f"Create a executor with max_workers={max_workers}." ) executor_id = _get_next_executor_id() _executor_kwargs = kwargs _executor = executor = cls( _executor_lock, max_workers=max_workers, executor_id=executor_id, **kwargs, ) else: if reuse == "auto": reuse = kwargs == _executor_kwargs if ( executor._flags.broken or executor._flags.shutdown or not reuse ): if executor._flags.broken: reason = "broken" elif executor._flags.shutdown: reason = "shutdown" else: reason = "arguments have changed" mp.util.debug( "Creating a new executor with max_workers=" f"{max_workers} as the previous instance cannot be " f"reused ({reason})." ) executor.shutdown(wait=True, kill_workers=kill_workers) _executor = executor = _executor_kwargs = None # Recursive call to build a new instance return cls.get_reusable_executor( max_workers=max_workers, **kwargs ) else: mp.util.debug( "Reusing existing executor with " f"max_workers={executor._max_workers}." ) is_reused = True executor._resize(max_workers) return executor, is_reused def submit(self, fn, *args, **kwargs): with self._submit_resize_lock: return super().submit(fn, *args, **kwargs) def _resize(self, max_workers): with self._submit_resize_lock: if max_workers is None: raise ValueError("Trying to resize with max_workers=None") elif max_workers == self._max_workers: return if self._executor_manager_thread is None: # If the executor_manager_thread has not been started # then no processes have been spawned and we can just # update _max_workers and return self._max_workers = max_workers return self._wait_job_completion() # Some process might have returned due to timeout so check how many # children are still alive. Use the _process_management_lock to # ensure that no process are spawned or timeout during the resize. with self._processes_management_lock: processes = list(self._processes.values()) nb_children_alive = sum(p.is_alive() for p in processes) self._max_workers = max_workers for _ in range(max_workers, nb_children_alive): self._call_queue.put(None) while ( len(self._processes) > max_workers and not self._flags.broken ): time.sleep(1e-3) self._adjust_process_count() processes = list(self._processes.values()) while not all(p.is_alive() for p in processes): time.sleep(1e-3) def _wait_job_completion(self): """Wait for the cache to be empty before resizing the pool.""" # Issue a warning to the user about the bad effect of this usage. if self._pending_work_items: warnings.warn( "Trying to resize an executor with running jobs: " "waiting for jobs completion before resizing.", UserWarning, ) mp.util.debug( f"Executor {self.executor_id} waiting for jobs completion " "before resizing" ) # Wait for the completion of the jobs while self._pending_work_items: time.sleep(1e-3) def _setup_queues(self, job_reducers, result_reducers): # As this executor can be resized, use a large queue size to avoid # underestimating capacity and introducing overhead queue_size = 2 * cpu_count() + EXTRA_QUEUED_CALLS super()._setup_queues( job_reducers, result_reducers, queue_size=queue_size ) joblib-1.3.2/joblib/externals/vendor_cloudpickle.sh000066400000000000000000000012371446465525000224510ustar00rootroot00000000000000#!/bin/sh # Script to do a local install of cloudpickle export LC_ALL=C INSTALL_FOLDER=tmp/cloudpickle_install rm -rf cloudpickle $INSTALL_FOLDER 2> /dev/null if [ -z "$1" ] then # Grab the latest stable release from PyPI CLOUDPICKLE=cloudpickle else CLOUDPICKLE=$1 fi pip install $CLOUDPICKLE --target $INSTALL_FOLDER cp -r $INSTALL_FOLDER/cloudpickle . rm -rf $INSTALL_FOLDER # Needed to rewrite the doctests # Note: BSD sed -i needs an argument unders OSX # so first renaming to .bak and then deleting backup files sed -i.bak "s/from cloudpickle.cloudpickle/from .cloudpickle/" cloudpickle/__init__.py find cloudpickle -name "*.bak" | xargs rm rm -r tmp joblib-1.3.2/joblib/externals/vendor_loky.sh000077500000000000000000000017051446465525000211340ustar00rootroot00000000000000#!/bin/sh # Script to do a local install of loky set +x export LC_ALL=C INSTALL_FOLDER=tmp/loky_install rm -rf loky $INSTALL_FOLDER 2> /dev/null if [ -z "$1" ] then # Grab the latest stable release from PyPI LOKY=loky else LOKY=$1 fi pip install --no-cache $LOKY --target $INSTALL_FOLDER cp -r $INSTALL_FOLDER/loky . rm -rf $INSTALL_FOLDER # Needed to rewrite the doctests # Note: BSD sed -i needs an argument unders OSX # so first renaming to .bak and then deleting backup files find loky -name "*.py" | xargs sed -i.bak "s/from loky/from joblib.externals.loky/" for f in $(git grep -l "cloudpickle" loky); do echo $f; sed -i.bak 's/import cloudpickle/from joblib.externals import cloudpickle/' $f sed -i.bak 's/from cloudpickle import/from joblib.externals.cloudpickle import/' $f done sed -i.bak "s/loky.backend.popen_loky/joblib.externals.loky.backend.popen_loky/" loky/backend/popen_loky_posix.py find loky -name "*.bak" | xargs rm joblib-1.3.2/joblib/func_inspect.py000066400000000000000000000335741446465525000173000ustar00rootroot00000000000000""" My own variation on function-specific inspect-like features. """ # Author: Gael Varoquaux # Copyright (c) 2009 Gael Varoquaux # License: BSD Style, 3 clauses. import inspect import warnings import re import os import collections from itertools import islice from tokenize import open as open_py_source from .logger import pformat full_argspec_fields = ('args varargs varkw defaults kwonlyargs ' 'kwonlydefaults annotations') full_argspec_type = collections.namedtuple('FullArgSpec', full_argspec_fields) def get_func_code(func): """ Attempts to retrieve a reliable function code hash. The reason we don't use inspect.getsource is that it caches the source, whereas we want this to be modified on the fly when the function is modified. Returns ------- func_code: string The function code source_file: string The path to the file in which the function is defined. first_line: int The first line of the code in the source file. Notes ------ This function does a bit more magic than inspect, and is thus more robust. """ source_file = None try: code = func.__code__ source_file = code.co_filename if not os.path.exists(source_file): # Use inspect for lambda functions and functions defined in an # interactive shell, or in doctests source_code = ''.join(inspect.getsourcelines(func)[0]) line_no = 1 if source_file.startswith('', source_file).groups() line_no = int(line_no) source_file = '' % source_file return source_code, source_file, line_no # Try to retrieve the source code. with open_py_source(source_file) as source_file_obj: first_line = code.co_firstlineno # All the lines after the function definition: source_lines = list(islice(source_file_obj, first_line - 1, None)) return ''.join(inspect.getblock(source_lines)), source_file, first_line except: # noqa: E722 # If the source code fails, we use the hash. This is fragile and # might change from one session to another. if hasattr(func, '__code__'): # Python 3.X return str(func.__code__.__hash__()), source_file, -1 else: # Weird objects like numpy ufunc don't have __code__ # This is fragile, as quite often the id of the object is # in the repr, so it might not persist across sessions, # however it will work for ufuncs. return repr(func), source_file, -1 def _clean_win_chars(string): """Windows cannot encode some characters in filename.""" import urllib if hasattr(urllib, 'quote'): quote = urllib.quote else: # In Python 3, quote is elsewhere import urllib.parse quote = urllib.parse.quote for char in ('<', '>', '!', ':', '\\'): string = string.replace(char, quote(char)) return string def get_func_name(func, resolv_alias=True, win_characters=True): """ Return the function import path (as a list of module names), and a name for the function. Parameters ---------- func: callable The func to inspect resolv_alias: boolean, optional If true, possible local aliases are indicated. win_characters: boolean, optional If true, substitute special characters using urllib.quote This is useful in Windows, as it cannot encode some filenames """ if hasattr(func, '__module__'): module = func.__module__ else: try: module = inspect.getmodule(func) except TypeError: if hasattr(func, '__class__'): module = func.__class__.__module__ else: module = 'unknown' if module is None: # Happens in doctests, eg module = '' if module == '__main__': try: filename = os.path.abspath(inspect.getsourcefile(func)) except: # noqa: E722 filename = None if filename is not None: # mangling of full path to filename parts = filename.split(os.sep) if parts[-1].startswith(', where: # - N is the cell number where the function was defined # - XYZ is a hash representing the function's code (and name). # It will be consistent across sessions and kernel restarts, # and will change if the function's code/name changes # We remove N so that cache is properly hit if the cell where # the func is defined is re-exectuted. # The XYZ hash should avoid collisions between functions with # the same name, both within the same notebook but also across # notebooks splitted = parts[-1].split('-') parts[-1] = '-'.join(splitted[:2] + splitted[3:]) elif len(parts) > 2 and parts[-2].startswith('ipykernel_'): # In a notebook session (ipykernel). Filename seems to be 'xyz' # of above. parts[-2] has the structure ipykernel_XXXXXX where # XXXXXX is a six-digit number identifying the current run (?). # If we split it off, the function again has the same # identifier across runs. parts[-2] = 'ipykernel' filename = '-'.join(parts) if filename.endswith('.py'): filename = filename[:-3] module = module + '-' + filename module = module.split('.') if hasattr(func, 'func_name'): name = func.func_name elif hasattr(func, '__name__'): name = func.__name__ else: name = 'unknown' # Hack to detect functions not defined at the module-level if resolv_alias: # TODO: Maybe add a warning here? if hasattr(func, 'func_globals') and name in func.func_globals: if not func.func_globals[name] is func: name = '%s-alias' % name if hasattr(func, '__qualname__') and func.__qualname__ != name: # Extend the module name in case of nested functions to avoid # (module, name) collisions module.extend(func.__qualname__.split(".")[:-1]) if inspect.ismethod(func): # We need to add the name of the class if hasattr(func, 'im_class'): klass = func.im_class module.append(klass.__name__) if os.name == 'nt' and win_characters: # Windows can't encode certain characters in filenames name = _clean_win_chars(name) module = [_clean_win_chars(s) for s in module] return module, name def _signature_str(function_name, arg_sig): """Helper function to output a function signature""" return '{}{}'.format(function_name, arg_sig) def _function_called_str(function_name, args, kwargs): """Helper function to output a function call""" template_str = '{0}({1}, {2})' args_str = repr(args)[1:-1] kwargs_str = ', '.join('%s=%s' % (k, v) for k, v in kwargs.items()) return template_str.format(function_name, args_str, kwargs_str) def filter_args(func, ignore_lst, args=(), kwargs=dict()): """ Filters the given args and kwargs using a list of arguments to ignore, and a function specification. Parameters ---------- func: callable Function giving the argument specification ignore_lst: list of strings List of arguments to ignore (either a name of an argument in the function spec, or '*', or '**') *args: list Positional arguments passed to the function. **kwargs: dict Keyword arguments passed to the function Returns ------- filtered_args: list List of filtered positional and keyword arguments. """ args = list(args) if isinstance(ignore_lst, str): # Catch a common mistake raise ValueError( 'ignore_lst must be a list of parameters to ignore ' '%s (type %s) was given' % (ignore_lst, type(ignore_lst))) # Special case for functools.partial objects if (not inspect.ismethod(func) and not inspect.isfunction(func)): if ignore_lst: warnings.warn('Cannot inspect object %s, ignore list will ' 'not work.' % func, stacklevel=2) return {'*': args, '**': kwargs} arg_sig = inspect.signature(func) arg_names = [] arg_defaults = [] arg_kwonlyargs = [] arg_varargs = None arg_varkw = None for param in arg_sig.parameters.values(): if param.kind is param.POSITIONAL_OR_KEYWORD: arg_names.append(param.name) elif param.kind is param.KEYWORD_ONLY: arg_names.append(param.name) arg_kwonlyargs.append(param.name) elif param.kind is param.VAR_POSITIONAL: arg_varargs = param.name elif param.kind is param.VAR_KEYWORD: arg_varkw = param.name if param.default is not param.empty: arg_defaults.append(param.default) if inspect.ismethod(func): # First argument is 'self', it has been removed by Python # we need to add it back: args = [func.__self__, ] + args # func is an instance method, inspect.signature(func) does not # include self, we need to fetch it from the class method, i.e # func.__func__ class_method_sig = inspect.signature(func.__func__) self_name = next(iter(class_method_sig.parameters)) arg_names = [self_name] + arg_names # XXX: Maybe I need an inspect.isbuiltin to detect C-level methods, such # as on ndarrays. _, name = get_func_name(func, resolv_alias=False) arg_dict = dict() arg_position = -1 for arg_position, arg_name in enumerate(arg_names): if arg_position < len(args): # Positional argument or keyword argument given as positional if arg_name not in arg_kwonlyargs: arg_dict[arg_name] = args[arg_position] else: raise ValueError( "Keyword-only parameter '%s' was passed as " 'positional parameter for %s:\n' ' %s was called.' % (arg_name, _signature_str(name, arg_sig), _function_called_str(name, args, kwargs)) ) else: position = arg_position - len(arg_names) if arg_name in kwargs: arg_dict[arg_name] = kwargs[arg_name] else: try: arg_dict[arg_name] = arg_defaults[position] except (IndexError, KeyError) as e: # Missing argument raise ValueError( 'Wrong number of arguments for %s:\n' ' %s was called.' % (_signature_str(name, arg_sig), _function_called_str(name, args, kwargs)) ) from e varkwargs = dict() for arg_name, arg_value in sorted(kwargs.items()): if arg_name in arg_dict: arg_dict[arg_name] = arg_value elif arg_varkw is not None: varkwargs[arg_name] = arg_value else: raise TypeError("Ignore list for %s() contains an unexpected " "keyword argument '%s'" % (name, arg_name)) if arg_varkw is not None: arg_dict['**'] = varkwargs if arg_varargs is not None: varargs = args[arg_position + 1:] arg_dict['*'] = varargs # Now remove the arguments to be ignored for item in ignore_lst: if item in arg_dict: arg_dict.pop(item) else: raise ValueError("Ignore list: argument '%s' is not defined for " "function %s" % (item, _signature_str(name, arg_sig)) ) # XXX: Return a sorted list of pairs? return arg_dict def _format_arg(arg): formatted_arg = pformat(arg, indent=2) if len(formatted_arg) > 1500: formatted_arg = '%s...' % formatted_arg[:700] return formatted_arg def format_signature(func, *args, **kwargs): # XXX: Should this use inspect.formatargvalues/formatargspec? module, name = get_func_name(func) module = [m for m in module if m] if module: module.append(name) module_path = '.'.join(module) else: module_path = name arg_str = list() previous_length = 0 for arg in args: formatted_arg = _format_arg(arg) if previous_length > 80: formatted_arg = '\n%s' % formatted_arg previous_length = len(formatted_arg) arg_str.append(formatted_arg) arg_str.extend(['%s=%s' % (v, _format_arg(i)) for v, i in kwargs.items()]) arg_str = ', '.join(arg_str) signature = '%s(%s)' % (name, arg_str) return module_path, signature def format_call(func, args, kwargs, object_name="Memory"): """ Returns a nicely formatted statement displaying the function call with the given arguments. """ path, signature = format_signature(func, *args, **kwargs) msg = '%s\n[%s] Calling %s...\n%s' % (80 * '_', object_name, path, signature) return msg # XXX: Not using logging framework # self.debug(msg) joblib-1.3.2/joblib/hashing.py000066400000000000000000000244471446465525000162400ustar00rootroot00000000000000""" Fast cryptographic hash of Python objects, with a special case for fast hashing of numpy arrays. """ # Author: Gael Varoquaux # Copyright (c) 2009 Gael Varoquaux # License: BSD Style, 3 clauses. import pickle import hashlib import sys import types import struct import io import decimal Pickler = pickle._Pickler class _ConsistentSet(object): """ Class used to ensure the hash of Sets is preserved whatever the order of its items. """ def __init__(self, set_sequence): # Forces order of elements in set to ensure consistent hash. try: # Trying first to order the set assuming the type of elements is # consistent and orderable. # This fails on python 3 when elements are unorderable # but we keep it in a try as it's faster. self._sequence = sorted(set_sequence) except (TypeError, decimal.InvalidOperation): # If elements are unorderable, sorting them using their hash. # This is slower but works in any case. self._sequence = sorted((hash(e) for e in set_sequence)) class _MyHash(object): """ Class used to hash objects that won't normally pickle """ def __init__(self, *args): self.args = args class Hasher(Pickler): """ A subclass of pickler, to do cryptographic hashing, rather than pickling. """ def __init__(self, hash_name='md5'): self.stream = io.BytesIO() # By default we want a pickle protocol that only changes with # the major python version and not the minor one protocol = 3 Pickler.__init__(self, self.stream, protocol=protocol) # Initialise the hash obj self._hash = hashlib.new(hash_name) def hash(self, obj, return_digest=True): try: self.dump(obj) except pickle.PicklingError as e: e.args += ('PicklingError while hashing %r: %r' % (obj, e),) raise dumps = self.stream.getvalue() self._hash.update(dumps) if return_digest: return self._hash.hexdigest() def save(self, obj): if isinstance(obj, (types.MethodType, type({}.pop))): # the Pickler cannot pickle instance methods; here we decompose # them into components that make them uniquely identifiable if hasattr(obj, '__func__'): func_name = obj.__func__.__name__ else: func_name = obj.__name__ inst = obj.__self__ if type(inst) is type(pickle): obj = _MyHash(func_name, inst.__name__) elif inst is None: # type(None) or type(module) do not pickle obj = _MyHash(func_name, inst) else: cls = obj.__self__.__class__ obj = _MyHash(func_name, inst, cls) Pickler.save(self, obj) def memoize(self, obj): # We want hashing to be sensitive to value instead of reference. # For example we want ['aa', 'aa'] and ['aa', 'aaZ'[:2]] # to hash to the same value and that's why we disable memoization # for strings if isinstance(obj, (bytes, str)): return Pickler.memoize(self, obj) # The dispatch table of the pickler is not accessible in Python # 3, as these lines are only bugware for IPython, we skip them. def save_global(self, obj, name=None, pack=struct.pack): # We have to override this method in order to deal with objects # defined interactively in IPython that are not injected in # __main__ kwargs = dict(name=name, pack=pack) del kwargs['pack'] try: Pickler.save_global(self, obj, **kwargs) except pickle.PicklingError: Pickler.save_global(self, obj, **kwargs) module = getattr(obj, "__module__", None) if module == '__main__': my_name = name if my_name is None: my_name = obj.__name__ mod = sys.modules[module] if not hasattr(mod, my_name): # IPython doesn't inject the variables define # interactively in __main__ setattr(mod, my_name, obj) dispatch = Pickler.dispatch.copy() # builtin dispatch[type(len)] = save_global # type dispatch[type(object)] = save_global # classobj dispatch[type(Pickler)] = save_global # function dispatch[type(pickle.dump)] = save_global def _batch_setitems(self, items): # forces order of keys in dict to ensure consistent hash. try: # Trying first to compare dict assuming the type of keys is # consistent and orderable. # This fails on python 3 when keys are unorderable # but we keep it in a try as it's faster. Pickler._batch_setitems(self, iter(sorted(items))) except TypeError: # If keys are unorderable, sorting them using their hash. This is # slower but works in any case. Pickler._batch_setitems(self, iter(sorted((hash(k), v) for k, v in items))) def save_set(self, set_items): # forces order of items in Set to ensure consistent hash Pickler.save(self, _ConsistentSet(set_items)) dispatch[type(set())] = save_set class NumpyHasher(Hasher): """ Special case the hasher for when numpy is loaded. """ def __init__(self, hash_name='md5', coerce_mmap=False): """ Parameters ---------- hash_name: string The hash algorithm to be used coerce_mmap: boolean Make no difference between np.memmap and np.ndarray objects. """ self.coerce_mmap = coerce_mmap Hasher.__init__(self, hash_name=hash_name) # delayed import of numpy, to avoid tight coupling import numpy as np self.np = np if hasattr(np, 'getbuffer'): self._getbuffer = np.getbuffer else: self._getbuffer = memoryview def save(self, obj): """ Subclass the save method, to hash ndarray subclass, rather than pickling them. Off course, this is a total abuse of the Pickler class. """ if isinstance(obj, self.np.ndarray) and not obj.dtype.hasobject: # Compute a hash of the object # The update function of the hash requires a c_contiguous buffer. if obj.shape == (): # 0d arrays need to be flattened because viewing them as bytes # raises a ValueError exception. obj_c_contiguous = obj.flatten() elif obj.flags.c_contiguous: obj_c_contiguous = obj elif obj.flags.f_contiguous: obj_c_contiguous = obj.T else: # Cater for non-single-segment arrays: this creates a # copy, and thus alleviates this issue. # XXX: There might be a more efficient way of doing this obj_c_contiguous = obj.flatten() # memoryview is not supported for some dtypes, e.g. datetime64, see # https://github.com/numpy/numpy/issues/4983. The # workaround is to view the array as bytes before # taking the memoryview. self._hash.update( self._getbuffer(obj_c_contiguous.view(self.np.uint8))) # We store the class, to be able to distinguish between # Objects with the same binary content, but different # classes. if self.coerce_mmap and isinstance(obj, self.np.memmap): # We don't make the difference between memmap and # normal ndarrays, to be able to reload previously # computed results with memmap. klass = self.np.ndarray else: klass = obj.__class__ # We also return the dtype and the shape, to distinguish # different views on the same data with different dtypes. # The object will be pickled by the pickler hashed at the end. obj = (klass, ('HASHED', obj.dtype, obj.shape, obj.strides)) elif isinstance(obj, self.np.dtype): # numpy.dtype consistent hashing is tricky to get right. This comes # from the fact that atomic np.dtype objects are interned: # ``np.dtype('f4') is np.dtype('f4')``. The situation is # complicated by the fact that this interning does not resist a # simple pickle.load/dump roundtrip: # ``pickle.loads(pickle.dumps(np.dtype('f4'))) is not # np.dtype('f4') Because pickle relies on memoization during # pickling, it is easy to # produce different hashes for seemingly identical objects, such as # ``[np.dtype('f4'), np.dtype('f4')]`` # and ``[np.dtype('f4'), pickle.loads(pickle.dumps('f4'))]``. # To prevent memoization from interfering with hashing, we isolate # the serialization (and thus the pickle memoization) of each dtype # using each time a different ``pickle.dumps`` call unrelated to # the current Hasher instance. self._hash.update("_HASHED_DTYPE".encode('utf-8')) self._hash.update(pickle.dumps(obj)) return Hasher.save(self, obj) def hash(obj, hash_name='md5', coerce_mmap=False): """ Quick calculation of a hash to identify uniquely Python objects containing numpy arrays. Parameters ---------- hash_name: 'md5' or 'sha1' Hashing algorithm used. sha1 is supposedly safer, but md5 is faster. coerce_mmap: boolean Make no difference between np.memmap and np.ndarray """ valid_hash_names = ('md5', 'sha1') if hash_name not in valid_hash_names: raise ValueError("Valid options for 'hash_name' are {}. " "Got hash_name={!r} instead." .format(valid_hash_names, hash_name)) if 'numpy' in sys.modules: hasher = NumpyHasher(hash_name=hash_name, coerce_mmap=coerce_mmap) else: hasher = Hasher(hash_name=hash_name) return hasher.hash(obj) joblib-1.3.2/joblib/logger.py000066400000000000000000000125271446465525000160720ustar00rootroot00000000000000""" Helpers for logging. This module needs much love to become useful. """ # Author: Gael Varoquaux # Copyright (c) 2008 Gael Varoquaux # License: BSD Style, 3 clauses. from __future__ import print_function import time import sys import os import shutil import logging import pprint from .disk import mkdirp def _squeeze_time(t): """Remove .1s to the time under Windows: this is the time it take to stat files. This is needed to make results similar to timings under Unix, for tests """ if sys.platform.startswith('win'): return max(0, t - .1) else: return t def format_time(t): t = _squeeze_time(t) return "%.1fs, %.1fmin" % (t, t / 60.) def short_format_time(t): t = _squeeze_time(t) if t > 60: return "%4.1fmin" % (t / 60.) else: return " %5.1fs" % (t) def pformat(obj, indent=0, depth=3): if 'numpy' in sys.modules: import numpy as np print_options = np.get_printoptions() np.set_printoptions(precision=6, threshold=64, edgeitems=1) else: print_options = None out = pprint.pformat(obj, depth=depth, indent=indent) if print_options: np.set_printoptions(**print_options) return out ############################################################################### # class `Logger` ############################################################################### class Logger(object): """ Base class for logging messages. """ def __init__(self, depth=3, name=None): """ Parameters ---------- depth: int, optional The depth of objects printed. name: str, optional The namespace to log to. If None, defaults to joblib. """ self.depth = depth self._name = name if name else 'joblib' def warn(self, msg): logging.getLogger(self._name).warning("[%s]: %s" % (self, msg)) def info(self, msg): logging.info("[%s]: %s" % (self, msg)) def debug(self, msg): # XXX: This conflicts with the debug flag used in children class logging.getLogger(self._name).debug("[%s]: %s" % (self, msg)) def format(self, obj, indent=0): """Return the formatted representation of the object.""" return pformat(obj, indent=indent, depth=self.depth) ############################################################################### # class `PrintTime` ############################################################################### class PrintTime(object): """ Print and log messages while keeping track of time. """ def __init__(self, logfile=None, logdir=None): if logfile is not None and logdir is not None: raise ValueError('Cannot specify both logfile and logdir') # XXX: Need argument docstring self.last_time = time.time() self.start_time = self.last_time if logdir is not None: logfile = os.path.join(logdir, 'joblib.log') self.logfile = logfile if logfile is not None: mkdirp(os.path.dirname(logfile)) if os.path.exists(logfile): # Rotate the logs for i in range(1, 9): try: shutil.move(logfile + '.%i' % i, logfile + '.%i' % (i + 1)) except: # noqa: E722 "No reason failing here" # Use a copy rather than a move, so that a process # monitoring this file does not get lost. try: shutil.copy(logfile, logfile + '.1') except: # noqa: E722 "No reason failing here" try: with open(logfile, 'w') as logfile: logfile.write('\nLogging joblib python script\n') logfile.write('\n---%s---\n' % time.ctime(self.last_time)) except: # noqa: E722 """ Multiprocessing writing to files can create race conditions. Rather fail silently than crash the computation. """ # XXX: We actually need a debug flag to disable this # silent failure. def __call__(self, msg='', total=False): """ Print the time elapsed between the last call and the current call, with an optional message. """ if not total: time_lapse = time.time() - self.last_time full_msg = "%s: %s" % (msg, format_time(time_lapse)) else: # FIXME: Too much logic duplicated time_lapse = time.time() - self.start_time full_msg = "%s: %.2fs, %.1f min" % (msg, time_lapse, time_lapse / 60) print(full_msg, file=sys.stderr) if self.logfile is not None: try: with open(self.logfile, 'a') as f: print(full_msg, file=f) except: # noqa: E722 """ Multiprocessing writing to files can create race conditions. Rather fail silently than crash the calculation. """ # XXX: We actually need a debug flag to disable this # silent failure. self.last_time = time.time() joblib-1.3.2/joblib/memory.py000066400000000000000000001341011446465525000161140ustar00rootroot00000000000000""" A context object for caching a function's return value each time it is called with the same input arguments. """ # Author: Gael Varoquaux # Copyright (c) 2009 Gael Varoquaux # License: BSD Style, 3 clauses. from __future__ import with_statement import logging import os from textwrap import dedent import time import pathlib import pydoc import re import functools import traceback import warnings import inspect import weakref from datetime import timedelta from tokenize import open as open_py_source # Local imports from . import hashing from .func_inspect import get_func_code, get_func_name, filter_args from .func_inspect import format_call from .func_inspect import format_signature from .logger import Logger, format_time, pformat from ._store_backends import StoreBackendBase, FileSystemStoreBackend from ._store_backends import CacheWarning # noqa FIRST_LINE_TEXT = "# first line:" # TODO: The following object should have a data store object as a sub # object, and the interface to persist and query should be separated in # the data store. # # This would enable creating 'Memory' objects with a different logic for # pickling that would simply span a MemorizedFunc with the same # store (or do we want to copy it to avoid cross-talks?), for instance to # implement HDF5 pickling. # TODO: Same remark for the logger, and probably use the Python logging # mechanism. def extract_first_line(func_code): """ Extract the first line information from the function code text if available. """ if func_code.startswith(FIRST_LINE_TEXT): func_code = func_code.split('\n') first_line = int(func_code[0][len(FIRST_LINE_TEXT):]) func_code = '\n'.join(func_code[1:]) else: first_line = -1 return func_code, first_line class JobLibCollisionWarning(UserWarning): """ Warn that there might be a collision between names of functions. """ _STORE_BACKENDS = {'local': FileSystemStoreBackend} def register_store_backend(backend_name, backend): """Extend available store backends. The Memory, MemorizeResult and MemorizeFunc objects are designed to be agnostic to the type of store used behind. By default, the local file system is used but this function gives the possibility to extend joblib's memory pattern with other types of storage such as cloud storage (S3, GCS, OpenStack, HadoopFS, etc) or blob DBs. Parameters ---------- backend_name: str The name identifying the store backend being registered. For example, 'local' is used with FileSystemStoreBackend. backend: StoreBackendBase subclass The name of a class that implements the StoreBackendBase interface. """ if not isinstance(backend_name, str): raise ValueError("Store backend name should be a string, " "'{0}' given.".format(backend_name)) if backend is None or not issubclass(backend, StoreBackendBase): raise ValueError("Store backend should inherit " "StoreBackendBase, " "'{0}' given.".format(backend)) _STORE_BACKENDS[backend_name] = backend def _store_backend_factory(backend, location, verbose=0, backend_options=None): """Return the correct store object for the given location.""" if backend_options is None: backend_options = {} if isinstance(location, pathlib.Path): location = str(location) if isinstance(location, StoreBackendBase): return location elif isinstance(location, str): obj = None location = os.path.expanduser(location) # The location is not a local file system, we look in the # registered backends if there's one matching the given backend # name. for backend_key, backend_obj in _STORE_BACKENDS.items(): if backend == backend_key: obj = backend_obj() # By default, we assume the FileSystemStoreBackend can be used if no # matching backend could be found. if obj is None: raise TypeError('Unknown location {0} or backend {1}'.format( location, backend)) # The store backend is configured with the extra named parameters, # some of them are specific to the underlying store backend. obj.configure(location, verbose=verbose, backend_options=backend_options) return obj elif location is not None: warnings.warn( "Instantiating a backend using a {} as a location is not " "supported by joblib. Returning None instead.".format( location.__class__.__name__), UserWarning) return None def _get_func_fullname(func): """Compute the part of part associated with a function.""" modules, funcname = get_func_name(func) modules.append(funcname) return os.path.join(*modules) def _build_func_identifier(func): """Build a roughly unique identifier for the cached function.""" parts = [] if isinstance(func, str): parts.append(func) else: parts.append(_get_func_fullname(func)) # We reuse historical fs-like way of building a function identifier return os.path.join(*parts) def _format_load_msg(func_id, args_id, timestamp=None, metadata=None): """ Helper function to format the message when loading the results. """ signature = "" try: if metadata is not None: args = ", ".join(['%s=%s' % (name, value) for name, value in metadata['input_args'].items()]) signature = "%s(%s)" % (os.path.basename(func_id), args) else: signature = os.path.basename(func_id) except KeyError: pass if timestamp is not None: ts_string = "{0: <16}".format(format_time(time.time() - timestamp)) else: ts_string = "" return '[Memory]{0}: Loading {1}'.format(ts_string, str(signature)) # An in-memory store to avoid looking at the disk-based function # source code to check if a function definition has changed _FUNCTION_HASHES = weakref.WeakKeyDictionary() ############################################################################### # class `MemorizedResult` ############################################################################### class MemorizedResult(Logger): """Object representing a cached value. Attributes ---------- location: str The location of joblib cache. Depends on the store backend used. func: function or str function whose output is cached. The string case is intended only for instantiation based on the output of repr() on another instance. (namely eval(repr(memorized_instance)) works). argument_hash: str hash of the function arguments. backend: str Type of store backend for reading/writing cache files. Default is 'local'. mmap_mode: {None, 'r+', 'r', 'w+', 'c'} The memmapping mode used when loading from cache numpy arrays. See numpy.load for the meaning of the different values. verbose: int verbosity level (0 means no message). timestamp, metadata: string for internal use only. """ def __init__(self, location, func, args_id, backend='local', mmap_mode=None, verbose=0, timestamp=None, metadata=None): Logger.__init__(self) self.func_id = _build_func_identifier(func) if isinstance(func, str): self.func = func else: self.func = self.func_id self.args_id = args_id self.store_backend = _store_backend_factory(backend, location, verbose=verbose) self.mmap_mode = mmap_mode if metadata is not None: self.metadata = metadata else: self.metadata = self.store_backend.get_metadata( [self.func_id, self.args_id]) self.duration = self.metadata.get('duration', None) self.verbose = verbose self.timestamp = timestamp @property def argument_hash(self): warnings.warn( "The 'argument_hash' attribute has been deprecated in version " "0.12 and will be removed in version 0.14.\n" "Use `args_id` attribute instead.", DeprecationWarning, stacklevel=2) return self.args_id def get(self): """Read value from cache and return it.""" if self.verbose: msg = _format_load_msg(self.func_id, self.args_id, timestamp=self.timestamp, metadata=self.metadata) else: msg = None try: return self.store_backend.load_item( [self.func_id, self.args_id], msg=msg, verbose=self.verbose) except ValueError as exc: new_exc = KeyError( "Error while trying to load a MemorizedResult's value. " "It seems that this folder is corrupted : {}".format( os.path.join( self.store_backend.location, self.func_id, self.args_id) )) raise new_exc from exc def clear(self): """Clear value from cache""" self.store_backend.clear_item([self.func_id, self.args_id]) def __repr__(self): return ('{class_name}(location="{location}", func="{func}", ' 'args_id="{args_id}")' .format(class_name=self.__class__.__name__, location=self.store_backend.location, func=self.func, args_id=self.args_id )) def __getstate__(self): state = self.__dict__.copy() state['timestamp'] = None return state class NotMemorizedResult(object): """Class representing an arbitrary value. This class is a replacement for MemorizedResult when there is no cache. """ __slots__ = ('value', 'valid') def __init__(self, value): self.value = value self.valid = True def get(self): if self.valid: return self.value else: raise KeyError("No value stored.") def clear(self): self.valid = False self.value = None def __repr__(self): if self.valid: return ('{class_name}({value})' .format(class_name=self.__class__.__name__, value=pformat(self.value))) else: return self.__class__.__name__ + ' with no value' # __getstate__ and __setstate__ are required because of __slots__ def __getstate__(self): return {"valid": self.valid, "value": self.value} def __setstate__(self, state): self.valid = state["valid"] self.value = state["value"] ############################################################################### # class `NotMemorizedFunc` ############################################################################### class NotMemorizedFunc(object): """No-op object decorating a function. This class replaces MemorizedFunc when there is no cache. It provides an identical API but does not write anything on disk. Attributes ---------- func: callable Original undecorated function. """ # Should be a light as possible (for speed) def __init__(self, func): self.func = func def __call__(self, *args, **kwargs): return self.func(*args, **kwargs) def call_and_shelve(self, *args, **kwargs): return NotMemorizedResult(self.func(*args, **kwargs)) def __repr__(self): return '{0}(func={1})'.format(self.__class__.__name__, self.func) def clear(self, warn=True): # Argument "warn" is for compatibility with MemorizedFunc.clear pass def call(self, *args, **kwargs): return self.func(*args, **kwargs) def check_call_in_cache(self, *args, **kwargs): return False ############################################################################### # class `MemorizedFunc` ############################################################################### class MemorizedFunc(Logger): """Callable object decorating a function for caching its return value each time it is called. Methods are provided to inspect the cache or clean it. Attributes ---------- func: callable The original, undecorated, function. location: string The location of joblib cache. Depends on the store backend used. backend: str Type of store backend for reading/writing cache files. Default is 'local', in which case the location is the path to a disk storage. ignore: list or None List of variable names to ignore when choosing whether to recompute. mmap_mode: {None, 'r+', 'r', 'w+', 'c'} The memmapping mode used when loading from cache numpy arrays. See numpy.load for the meaning of the different values. compress: boolean, or integer Whether to zip the stored data on disk. If an integer is given, it should be between 1 and 9, and sets the amount of compression. Note that compressed arrays cannot be read by memmapping. verbose: int, optional The verbosity flag, controls messages that are issued as the function is evaluated. cache_validation_callback: callable, optional Callable to check if a result in cache is valid or is to be recomputed. When the function is called with arguments for which a cache exists, the callback is called with the cache entry's metadata as its sole argument. If it returns True, the cached result is returned, else the cache for these arguments is cleared and the result is recomputed. """ # ------------------------------------------------------------------------ # Public interface # ------------------------------------------------------------------------ def __init__(self, func, location, backend='local', ignore=None, mmap_mode=None, compress=False, verbose=1, timestamp=None, cache_validation_callback=None): Logger.__init__(self) self.mmap_mode = mmap_mode self.compress = compress self.func = func self.cache_validation_callback = cache_validation_callback if ignore is None: ignore = [] self.ignore = ignore self._verbose = verbose # retrieve store object from backend type and location. self.store_backend = _store_backend_factory(backend, location, verbose=verbose, backend_options=dict( compress=compress, mmap_mode=mmap_mode), ) if self.store_backend is not None: # Create func directory on demand. self.store_backend.store_cached_func_code([ _build_func_identifier(self.func) ]) if timestamp is None: timestamp = time.time() self.timestamp = timestamp try: functools.update_wrapper(self, func) except Exception: " Objects like ufunc don't like that " if inspect.isfunction(func): doc = pydoc.TextDoc().document(func) # Remove blank line doc = doc.replace('\n', '\n\n', 1) # Strip backspace-overprints for compatibility with autodoc doc = re.sub('\x08.', '', doc) else: # Pydoc does a poor job on other objects doc = func.__doc__ self.__doc__ = 'Memoized version of %s' % doc self._func_code_info = None self._func_code_id = None def _is_in_cache_and_valid(self, path): """Check if the function call is cached and valid for given arguments. - Compare the function code with the one from the cached function, asserting if it has changed. - Check if the function call is present in the cache. - Call `cache_validation_callback` for user define cache validation. Returns True if the function call is in cache and can be used, and returns False otherwise. """ # Check if the code of the function has changed if not self._check_previous_func_code(stacklevel=4): return False # Check if this specific call is in the cache if not self.store_backend.contains_item(path): return False # Call the user defined cache validation callback metadata = self.store_backend.get_metadata(path) if (self.cache_validation_callback is not None and not self.cache_validation_callback(metadata)): self.store_backend.clear_item(path) return False return True def _cached_call(self, args, kwargs, shelving=False): """Call wrapped function and cache result, or read cache if available. This function returns the wrapped function output and some metadata. Arguments: ---------- args, kwargs: list and dict input arguments for wrapped function shelving: bool True when called via the call_and_shelve function. Returns ------- output: value or tuple or None Output of the wrapped function. If shelving is True and the call has been already cached, output is None. argument_hash: string Hash of function arguments. metadata: dict Some metadata about wrapped function call (see _persist_input()). """ func_id, args_id = self._get_output_identifiers(*args, **kwargs) metadata = None msg = None # Whether or not the memorized function must be called must_call = False if self._verbose >= 20: logging.basicConfig(level=logging.INFO) _, name = get_func_name(self.func) location = self.store_backend.get_cached_func_info([func_id])[ 'location'] _, signature = format_signature(self.func, *args, **kwargs) self.info( dedent( f""" Querying {name} with signature {signature}. (argument hash {args_id}) The store location is {location}. """ ) ) # Compare the function code with the previous to see if the # function code has changed and check if the results are present in # the cache. if self._is_in_cache_and_valid([func_id, args_id]): try: t0 = time.time() if self._verbose: msg = _format_load_msg(func_id, args_id, timestamp=self.timestamp, metadata=metadata) if not shelving: # When shelving, we do not need to load the output out = self.store_backend.load_item( [func_id, args_id], msg=msg, verbose=self._verbose) else: out = None if self._verbose > 4: t = time.time() - t0 _, name = get_func_name(self.func) msg = '%s cache loaded - %s' % (name, format_time(t)) print(max(0, (80 - len(msg))) * '_' + msg) except Exception: # XXX: Should use an exception logger _, signature = format_signature(self.func, *args, **kwargs) self.warn('Exception while loading results for ' '{}\n {}'.format(signature, traceback.format_exc())) must_call = True else: if self._verbose > 10: _, name = get_func_name(self.func) self.warn('Computing func {0}, argument hash {1} ' 'in location {2}' .format(name, args_id, self.store_backend. get_cached_func_info([func_id])['location'])) must_call = True if must_call: out, metadata = self.call(*args, **kwargs) if self.mmap_mode is not None: # Memmap the output at the first call to be consistent with # later calls if self._verbose: msg = _format_load_msg(func_id, args_id, timestamp=self.timestamp, metadata=metadata) out = self.store_backend.load_item([func_id, args_id], msg=msg, verbose=self._verbose) return (out, args_id, metadata) @property def func_code_info(self): # 3-tuple property containing: the function source code, source file, # and first line of the code inside the source file if hasattr(self.func, '__code__'): if self._func_code_id is None: self._func_code_id = id(self.func.__code__) elif id(self.func.__code__) != self._func_code_id: # Be robust to dynamic reassignments of self.func.__code__ self._func_code_info = None if self._func_code_info is None: # Cache the source code of self.func . Provided that get_func_code # (which should be called once on self) gets called in the process # in which self.func was defined, this caching mechanism prevents # undesired cache clearing when the cached function is called in # an environment where the introspection utilities get_func_code # relies on do not work (typically, in joblib child processes). # See #1035 for more info # TODO (pierreglaser): do the same with get_func_name? self._func_code_info = get_func_code(self.func) return self._func_code_info def call_and_shelve(self, *args, **kwargs): """Call wrapped function, cache result and return a reference. This method returns a reference to the cached result instead of the result itself. The reference object is small and pickeable, allowing to send or store it easily. Call .get() on reference object to get result. Returns ------- cached_result: MemorizedResult or NotMemorizedResult reference to the value returned by the wrapped function. The class "NotMemorizedResult" is used when there is no cache activated (e.g. location=None in Memory). """ _, args_id, metadata = self._cached_call(args, kwargs, shelving=True) return MemorizedResult(self.store_backend, self.func, args_id, metadata=metadata, verbose=self._verbose - 1, timestamp=self.timestamp) def __call__(self, *args, **kwargs): return self._cached_call(args, kwargs)[0] def __getstate__(self): # Make sure self.func's source is introspected prior to being pickled - # code introspection utilities typically do not work inside child # processes _ = self.func_code_info # We don't store the timestamp when pickling, to avoid the hash # depending from it. state = self.__dict__.copy() state['timestamp'] = None # Invalidate the code id as id(obj) will be different in the child state['_func_code_id'] = None return state def check_call_in_cache(self, *args, **kwargs): """Check if function call is in the memory cache. Does not call the function or do any work besides func inspection and arg hashing. Returns ------- is_call_in_cache: bool Whether or not the result of the function has been cached for the input arguments that have been passed. """ func_id, args_id = self._get_output_identifiers(*args, **kwargs) return self.store_backend.contains_item((func_id, args_id)) # ------------------------------------------------------------------------ # Private interface # ------------------------------------------------------------------------ def _get_argument_hash(self, *args, **kwargs): return hashing.hash(filter_args(self.func, self.ignore, args, kwargs), coerce_mmap=(self.mmap_mode is not None)) def _get_output_identifiers(self, *args, **kwargs): """Return the func identifier and input parameter hash of a result.""" func_id = _build_func_identifier(self.func) argument_hash = self._get_argument_hash(*args, **kwargs) return func_id, argument_hash def _hash_func(self): """Hash a function to key the online cache""" func_code_h = hash(getattr(self.func, '__code__', None)) return id(self.func), hash(self.func), func_code_h def _write_func_code(self, func_code, first_line): """ Write the function code and the filename to a file. """ # We store the first line because the filename and the function # name is not always enough to identify a function: people # sometimes have several functions named the same way in a # file. This is bad practice, but joblib should be robust to bad # practice. func_id = _build_func_identifier(self.func) func_code = u'%s %i\n%s' % (FIRST_LINE_TEXT, first_line, func_code) self.store_backend.store_cached_func_code([func_id], func_code) # Also store in the in-memory store of function hashes is_named_callable = False is_named_callable = (hasattr(self.func, '__name__') and self.func.__name__ != '') if is_named_callable: # Don't do this for lambda functions or strange callable # objects, as it ends up being too fragile func_hash = self._hash_func() try: _FUNCTION_HASHES[self.func] = func_hash except TypeError: # Some callable are not hashable pass def _check_previous_func_code(self, stacklevel=2): """ stacklevel is the depth a which this function is called, to issue useful warnings to the user. """ # First check if our function is in the in-memory store. # Using the in-memory store not only makes things faster, but it # also renders us robust to variations of the files when the # in-memory version of the code does not vary try: if self.func in _FUNCTION_HASHES: # We use as an identifier the id of the function and its # hash. This is more likely to falsely change than have hash # collisions, thus we are on the safe side. func_hash = self._hash_func() if func_hash == _FUNCTION_HASHES[self.func]: return True except TypeError: # Some callables are not hashable pass # Here, we go through some effort to be robust to dynamically # changing code and collision. We cannot inspect.getsource # because it is not reliable when using IPython's magic "%run". func_code, source_file, first_line = self.func_code_info func_id = _build_func_identifier(self.func) try: old_func_code, old_first_line =\ extract_first_line( self.store_backend.get_cached_func_code([func_id])) except (IOError, OSError): # some backend can also raise OSError self._write_func_code(func_code, first_line) return False if old_func_code == func_code: return True # We have differing code, is this because we are referring to # different functions, or because the function we are referring to has # changed? _, func_name = get_func_name(self.func, resolv_alias=False, win_characters=False) if old_first_line == first_line == -1 or func_name == '': if not first_line == -1: func_description = ("{0} ({1}:{2})" .format(func_name, source_file, first_line)) else: func_description = func_name warnings.warn(JobLibCollisionWarning( "Cannot detect name collisions for function '{0}'" .format(func_description)), stacklevel=stacklevel) # Fetch the code at the old location and compare it. If it is the # same than the code store, we have a collision: the code in the # file has not changed, but the name we have is pointing to a new # code block. if not old_first_line == first_line and source_file is not None: possible_collision = False if os.path.exists(source_file): _, func_name = get_func_name(self.func, resolv_alias=False) num_lines = len(func_code.split('\n')) with open_py_source(source_file) as f: on_disk_func_code = f.readlines()[ old_first_line - 1:old_first_line - 1 + num_lines - 1] on_disk_func_code = ''.join(on_disk_func_code) possible_collision = (on_disk_func_code.rstrip() == old_func_code.rstrip()) else: possible_collision = source_file.startswith(' 10: _, func_name = get_func_name(self.func, resolv_alias=False) self.warn("Function {0} (identified by {1}) has changed" ".".format(func_name, func_id)) self.clear(warn=True) return False def clear(self, warn=True): """Empty the function's cache.""" func_id = _build_func_identifier(self.func) if self._verbose > 0 and warn: self.warn("Clearing function cache identified by %s" % func_id) self.store_backend.clear_path([func_id, ]) func_code, _, first_line = self.func_code_info self._write_func_code(func_code, first_line) def call(self, *args, **kwargs): """Force the execution of the function with the given arguments. The output values will be persisted, i.e., the cache will be updated with any new values. Parameters ---------- *args: arguments The arguments. **kwargs: keyword arguments Keyword arguments. Returns ------- output : object The output of the function call. metadata : dict The metadata associated with the call. """ start_time = time.time() func_id, args_id = self._get_output_identifiers(*args, **kwargs) if self._verbose > 0: print(format_call(self.func, args, kwargs)) output = self.func(*args, **kwargs) self.store_backend.dump_item( [func_id, args_id], output, verbose=self._verbose) duration = time.time() - start_time metadata = self._persist_input(duration, args, kwargs) if self._verbose > 0: _, name = get_func_name(self.func) msg = '%s - %s' % (name, format_time(duration)) print(max(0, (80 - len(msg))) * '_' + msg) return output, metadata def _persist_input(self, duration, args, kwargs, this_duration_limit=0.5): """ Save a small summary of the call using json format in the output directory. output_dir: string directory where to write metadata. duration: float time taken by hashing input arguments, calling the wrapped function and persisting its output. args, kwargs: list and dict input arguments for wrapped function this_duration_limit: float Max execution time for this function before issuing a warning. """ start_time = time.time() argument_dict = filter_args(self.func, self.ignore, args, kwargs) input_repr = dict((k, repr(v)) for k, v in argument_dict.items()) # This can fail due to race-conditions with multiple # concurrent joblibs removing the file or the directory metadata = { "duration": duration, "input_args": input_repr, "time": start_time, } func_id, args_id = self._get_output_identifiers(*args, **kwargs) self.store_backend.store_metadata([func_id, args_id], metadata) this_duration = time.time() - start_time if this_duration > this_duration_limit: # This persistence should be fast. It will not be if repr() takes # time and its output is large, because json.dump will have to # write a large file. This should not be an issue with numpy arrays # for which repr() always output a short representation, but can # be with complex dictionaries. Fixing the problem should be a # matter of replacing repr() above by something smarter. warnings.warn("Persisting input arguments took %.2fs to run." "If this happens often in your code, it can cause " "performance problems " "(results will be correct in all cases). " "The reason for this is probably some large input " "arguments for a wrapped function." % this_duration, stacklevel=5) return metadata # ------------------------------------------------------------------------ # Private `object` interface # ------------------------------------------------------------------------ def __repr__(self): return '{class_name}(func={func}, location={location})'.format( class_name=self.__class__.__name__, func=self.func, location=self.store_backend.location,) ############################################################################### # class `Memory` ############################################################################### class Memory(Logger): """ A context object for caching a function's return value each time it is called with the same input arguments. All values are cached on the filesystem, in a deep directory structure. Read more in the :ref:`User Guide `. Parameters ---------- location: str, pathlib.Path or None The path of the base directory to use as a data store or None. If None is given, no caching is done and the Memory object is completely transparent. This option replaces cachedir since version 0.12. backend: str, optional Type of store backend for reading/writing cache files. Default: 'local'. The 'local' backend is using regular filesystem operations to manipulate data (open, mv, etc) in the backend. mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional The memmapping mode used when loading from cache numpy arrays. See numpy.load for the meaning of the arguments. compress: boolean, or integer, optional Whether to zip the stored data on disk. If an integer is given, it should be between 1 and 9, and sets the amount of compression. Note that compressed arrays cannot be read by memmapping. verbose: int, optional Verbosity flag, controls the debug messages that are issued as functions are evaluated. bytes_limit: int | str, optional Limit in bytes of the size of the cache. By default, the size of the cache is unlimited. When reducing the size of the cache, ``joblib`` keeps the most recently accessed items first. If a str is passed, it is converted to a number of bytes using units { K | M | G} for kilo, mega, giga. **Note:** You need to call :meth:`joblib.Memory.reduce_size` to actually reduce the cache size to be less than ``bytes_limit``. **Note:** This argument has been deprecated. One should give the value of ``bytes_limit`` directly in :meth:`joblib.Memory.reduce_size`. backend_options: dict, optional Contains a dictionary of named parameters used to configure the store backend. """ # ------------------------------------------------------------------------ # Public interface # ------------------------------------------------------------------------ def __init__(self, location=None, backend='local', mmap_mode=None, compress=False, verbose=1, bytes_limit=None, backend_options=None): Logger.__init__(self) self._verbose = verbose self.mmap_mode = mmap_mode self.timestamp = time.time() if bytes_limit is not None: warnings.warn( "bytes_limit argument has been deprecated. It will be removed " "in version 1.5. Please pass its value directly to " "Memory.reduce_size.", category=DeprecationWarning ) self.bytes_limit = bytes_limit self.backend = backend self.compress = compress if backend_options is None: backend_options = {} self.backend_options = backend_options if compress and mmap_mode is not None: warnings.warn('Compressed results cannot be memmapped', stacklevel=2) self.location = location if isinstance(location, str): location = os.path.join(location, 'joblib') self.store_backend = _store_backend_factory( backend, location, verbose=self._verbose, backend_options=dict(compress=compress, mmap_mode=mmap_mode, **backend_options)) def cache(self, func=None, ignore=None, verbose=None, mmap_mode=False, cache_validation_callback=None): """ Decorates the given function func to only compute its return value for input arguments not cached on disk. Parameters ---------- func: callable, optional The function to be decorated ignore: list of strings A list of arguments name to ignore in the hashing verbose: integer, optional The verbosity mode of the function. By default that of the memory object is used. mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional The memmapping mode used when loading from cache numpy arrays. See numpy.load for the meaning of the arguments. By default that of the memory object is used. cache_validation_callback: callable, optional Callable to validate whether or not the cache is valid. When the cached function is called with arguments for which a cache exists, this callable is called with the metadata of the cached result as its sole argument. If it returns True, then the cached result is returned, else the cache for these arguments is cleared and recomputed. Returns ------- decorated_func: MemorizedFunc object The returned object is a MemorizedFunc object, that is callable (behaves like a function), but offers extra methods for cache lookup and management. See the documentation for :class:`joblib.memory.MemorizedFunc`. """ if (cache_validation_callback is not None and not callable(cache_validation_callback)): raise ValueError( "cache_validation_callback needs to be callable. " f"Got {cache_validation_callback}." ) if func is None: # Partial application, to be able to specify extra keyword # arguments in decorators return functools.partial( self.cache, ignore=ignore, mmap_mode=mmap_mode, verbose=verbose, cache_validation_callback=cache_validation_callback ) if self.store_backend is None: return NotMemorizedFunc(func) if verbose is None: verbose = self._verbose if mmap_mode is False: mmap_mode = self.mmap_mode if isinstance(func, MemorizedFunc): func = func.func return MemorizedFunc( func, location=self.store_backend, backend=self.backend, ignore=ignore, mmap_mode=mmap_mode, compress=self.compress, verbose=verbose, timestamp=self.timestamp, cache_validation_callback=cache_validation_callback ) def clear(self, warn=True): """ Erase the complete cache directory. """ if warn: self.warn('Flushing completely the cache') if self.store_backend is not None: self.store_backend.clear() # As the cache is completely clear, make sure the _FUNCTION_HASHES # cache is also reset. Else, for a function that is present in this # table, results cached after this clear will be have cache miss # as the function code is not re-written. _FUNCTION_HASHES.clear() def reduce_size(self, bytes_limit=None, items_limit=None, age_limit=None): """Remove cache elements to make the cache fit its limits. The limitation can impose that the cache size fits in ``bytes_limit``, that the number of cache items is no more than ``items_limit``, and that all files in cache are not older than ``age_limit``. Parameters ---------- bytes_limit: int | str, optional Limit in bytes of the size of the cache. By default, the size of the cache is unlimited. When reducing the size of the cache, ``joblib`` keeps the most recently accessed items first. If a str is passed, it is converted to a number of bytes using units { K | M | G} for kilo, mega, giga. items_limit: int, optional Number of items to limit the cache to. By default, the number of items in the cache is unlimited. When reducing the size of the cache, ``joblib`` keeps the most recently accessed items first. age_limit: datetime.timedelta, optional Maximum age of items to limit the cache to. When reducing the size of the cache, any items last accessed more than the given length of time ago are deleted. """ if bytes_limit is None: bytes_limit = self.bytes_limit if self.store_backend is None: # No cached results, this function does nothing. return if bytes_limit is None and items_limit is None and age_limit is None: # No limitation to impose, returning return # Defers the actual limits enforcing to the store backend. self.store_backend.enforce_store_limits( bytes_limit, items_limit, age_limit ) def eval(self, func, *args, **kwargs): """ Eval function func with arguments `*args` and `**kwargs`, in the context of the memory. This method works similarly to the builtin `apply`, except that the function is called only if the cache is not up to date. """ if self.store_backend is None: return func(*args, **kwargs) return self.cache(func)(*args, **kwargs) # ------------------------------------------------------------------------ # Private `object` interface # ------------------------------------------------------------------------ def __repr__(self): return '{class_name}(location={location})'.format( class_name=self.__class__.__name__, location=(None if self.store_backend is None else self.store_backend.location)) def __getstate__(self): """ We don't store the timestamp when pickling, to avoid the hash depending from it. """ state = self.__dict__.copy() state['timestamp'] = None return state ############################################################################### # cache_validation_callback helpers ############################################################################### def expires_after(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0): """Helper cache_validation_callback to force recompute after a duration. Parameters ---------- days, seconds, microseconds, milliseconds, minutes, hours, weeks: numbers argument passed to a timedelta. """ delta = timedelta( days=days, seconds=seconds, microseconds=microseconds, milliseconds=milliseconds, minutes=minutes, hours=hours, weeks=weeks ) def cache_validation_callback(metadata): computation_age = time.time() - metadata['time'] return computation_age < delta.total_seconds() return cache_validation_callback joblib-1.3.2/joblib/numpy_pickle.py000066400000000000000000000644061446465525000173150ustar00rootroot00000000000000"""Utilities for fast persistence of big data, with optional compression.""" # Author: Gael Varoquaux # Copyright (c) 2009 Gael Varoquaux # License: BSD Style, 3 clauses. import pickle import os import warnings import io from pathlib import Path from .compressor import lz4, LZ4_NOT_INSTALLED_ERROR from .compressor import _COMPRESSORS, register_compressor, BinaryZlibFile from .compressor import (ZlibCompressorWrapper, GzipCompressorWrapper, BZ2CompressorWrapper, LZMACompressorWrapper, XZCompressorWrapper, LZ4CompressorWrapper) from .numpy_pickle_utils import Unpickler, Pickler from .numpy_pickle_utils import _read_fileobject, _write_fileobject from .numpy_pickle_utils import _read_bytes, BUFFER_SIZE from .numpy_pickle_utils import _ensure_native_byte_order from .numpy_pickle_compat import load_compatibility from .numpy_pickle_compat import NDArrayWrapper # For compatibility with old versions of joblib, we need ZNDArrayWrapper # to be visible in the current namespace. # Explicitly skipping next line from flake8 as it triggers an F401 warning # which we don't care. from .numpy_pickle_compat import ZNDArrayWrapper # noqa from .backports import make_memmap # Register supported compressors register_compressor('zlib', ZlibCompressorWrapper()) register_compressor('gzip', GzipCompressorWrapper()) register_compressor('bz2', BZ2CompressorWrapper()) register_compressor('lzma', LZMACompressorWrapper()) register_compressor('xz', XZCompressorWrapper()) register_compressor('lz4', LZ4CompressorWrapper()) ############################################################################### # Utility objects for persistence. # For convenience, 16 bytes are used to be sure to cover all the possible # dtypes' alignments. For reference, see: # https://numpy.org/devdocs/dev/alignment.html NUMPY_ARRAY_ALIGNMENT_BYTES = 16 class NumpyArrayWrapper(object): """An object to be persisted instead of numpy arrays. This object is used to hack into the pickle machinery and read numpy array data from our custom persistence format. More precisely, this object is used for: * carrying the information of the persisted array: subclass, shape, order, dtype. Those ndarray metadata are used to correctly reconstruct the array with low level numpy functions. * determining if memmap is allowed on the array. * reading the array bytes from a file. * reading the array using memorymap from a file. * writing the array bytes to a file. Attributes ---------- subclass: numpy.ndarray subclass Determine the subclass of the wrapped array. shape: numpy.ndarray shape Determine the shape of the wrapped array. order: {'C', 'F'} Determine the order of wrapped array data. 'C' is for C order, 'F' is for fortran order. dtype: numpy.ndarray dtype Determine the data type of the wrapped array. allow_mmap: bool Determine if memory mapping is allowed on the wrapped array. Default: False. """ def __init__(self, subclass, shape, order, dtype, allow_mmap=False, numpy_array_alignment_bytes=NUMPY_ARRAY_ALIGNMENT_BYTES): """Constructor. Store the useful information for later.""" self.subclass = subclass self.shape = shape self.order = order self.dtype = dtype self.allow_mmap = allow_mmap # We make numpy_array_alignment_bytes an instance attribute to allow us # to change our mind about the default alignment and still load the old # pickles (with the previous alignment) correctly self.numpy_array_alignment_bytes = numpy_array_alignment_bytes def safe_get_numpy_array_alignment_bytes(self): # NumpyArrayWrapper instances loaded from joblib <= 1.1 pickles don't # have an numpy_array_alignment_bytes attribute return getattr(self, 'numpy_array_alignment_bytes', None) def write_array(self, array, pickler): """Write array bytes to pickler file handle. This function is an adaptation of the numpy write_array function available in version 1.10.1 in numpy/lib/format.py. """ # Set buffer size to 16 MiB to hide the Python loop overhead. buffersize = max(16 * 1024 ** 2 // array.itemsize, 1) if array.dtype.hasobject: # We contain Python objects so we cannot write out the data # directly. Instead, we will pickle it out with version 2 of the # pickle protocol. pickle.dump(array, pickler.file_handle, protocol=2) else: numpy_array_alignment_bytes = \ self.safe_get_numpy_array_alignment_bytes() if numpy_array_alignment_bytes is not None: current_pos = pickler.file_handle.tell() pos_after_padding_byte = current_pos + 1 padding_length = numpy_array_alignment_bytes - ( pos_after_padding_byte % numpy_array_alignment_bytes) # A single byte is written that contains the padding length in # bytes padding_length_byte = int.to_bytes( padding_length, length=1, byteorder='little') pickler.file_handle.write(padding_length_byte) if padding_length != 0: padding = b'\xff' * padding_length pickler.file_handle.write(padding) for chunk in pickler.np.nditer(array, flags=['external_loop', 'buffered', 'zerosize_ok'], buffersize=buffersize, order=self.order): pickler.file_handle.write(chunk.tobytes('C')) def read_array(self, unpickler): """Read array from unpickler file handle. This function is an adaptation of the numpy read_array function available in version 1.10.1 in numpy/lib/format.py. """ if len(self.shape) == 0: count = 1 else: # joblib issue #859: we cast the elements of self.shape to int64 to # prevent a potential overflow when computing their product. shape_int64 = [unpickler.np.int64(x) for x in self.shape] count = unpickler.np.multiply.reduce(shape_int64) # Now read the actual data. if self.dtype.hasobject: # The array contained Python objects. We need to unpickle the data. array = pickle.load(unpickler.file_handle) else: numpy_array_alignment_bytes = \ self.safe_get_numpy_array_alignment_bytes() if numpy_array_alignment_bytes is not None: padding_byte = unpickler.file_handle.read(1) padding_length = int.from_bytes( padding_byte, byteorder='little') if padding_length != 0: unpickler.file_handle.read(padding_length) # This is not a real file. We have to read it the # memory-intensive way. # crc32 module fails on reads greater than 2 ** 32 bytes, # breaking large reads from gzip streams. Chunk reads to # BUFFER_SIZE bytes to avoid issue and reduce memory overhead # of the read. In non-chunked case count < max_read_count, so # only one read is performed. max_read_count = BUFFER_SIZE // min(BUFFER_SIZE, self.dtype.itemsize) array = unpickler.np.empty(count, dtype=self.dtype) for i in range(0, count, max_read_count): read_count = min(max_read_count, count - i) read_size = int(read_count * self.dtype.itemsize) data = _read_bytes(unpickler.file_handle, read_size, "array data") array[i:i + read_count] = \ unpickler.np.frombuffer(data, dtype=self.dtype, count=read_count) del data if self.order == 'F': array.shape = self.shape[::-1] array = array.transpose() else: array.shape = self.shape # Detect byte order mismatch and swap as needed. return _ensure_native_byte_order(array) def read_mmap(self, unpickler): """Read an array using numpy memmap.""" current_pos = unpickler.file_handle.tell() offset = current_pos numpy_array_alignment_bytes = \ self.safe_get_numpy_array_alignment_bytes() if numpy_array_alignment_bytes is not None: padding_byte = unpickler.file_handle.read(1) padding_length = int.from_bytes(padding_byte, byteorder='little') # + 1 is for the padding byte offset += padding_length + 1 if unpickler.mmap_mode == 'w+': unpickler.mmap_mode = 'r+' marray = make_memmap(unpickler.filename, dtype=self.dtype, shape=self.shape, order=self.order, mode=unpickler.mmap_mode, offset=offset) # update the offset so that it corresponds to the end of the read array unpickler.file_handle.seek(offset + marray.nbytes) if (numpy_array_alignment_bytes is None and current_pos % NUMPY_ARRAY_ALIGNMENT_BYTES != 0): message = ( f'The memmapped array {marray} loaded from the file ' f'{unpickler.file_handle.name} is not byte aligned. ' 'This may cause segmentation faults if this memmapped array ' 'is used in some libraries like BLAS or PyTorch. ' 'To get rid of this warning, regenerate your pickle file ' 'with joblib >= 1.2.0. ' 'See https://github.com/joblib/joblib/issues/563 ' 'for more details' ) warnings.warn(message) return _ensure_native_byte_order(marray) def read(self, unpickler): """Read the array corresponding to this wrapper. Use the unpickler to get all information to correctly read the array. Parameters ---------- unpickler: NumpyUnpickler Returns ------- array: numpy.ndarray """ # When requested, only use memmap mode if allowed. if unpickler.mmap_mode is not None and self.allow_mmap: array = self.read_mmap(unpickler) else: array = self.read_array(unpickler) # Manage array subclass case if (hasattr(array, '__array_prepare__') and self.subclass not in (unpickler.np.ndarray, unpickler.np.memmap)): # We need to reconstruct another subclass new_array = unpickler.np.core.multiarray._reconstruct( self.subclass, (0,), 'b') return new_array.__array_prepare__(array) else: return array ############################################################################### # Pickler classes class NumpyPickler(Pickler): """A pickler to persist big data efficiently. The main features of this object are: * persistence of numpy arrays in a single file. * optional compression with a special care on avoiding memory copies. Attributes ---------- fp: file File object handle used for serializing the input object. protocol: int, optional Pickle protocol used. Default is pickle.DEFAULT_PROTOCOL. """ dispatch = Pickler.dispatch.copy() def __init__(self, fp, protocol=None): self.file_handle = fp self.buffered = isinstance(self.file_handle, BinaryZlibFile) # By default we want a pickle protocol that only changes with # the major python version and not the minor one if protocol is None: protocol = pickle.DEFAULT_PROTOCOL Pickler.__init__(self, self.file_handle, protocol=protocol) # delayed import of numpy, to avoid tight coupling try: import numpy as np except ImportError: np = None self.np = np def _create_array_wrapper(self, array): """Create and returns a numpy array wrapper from a numpy array.""" order = 'F' if (array.flags.f_contiguous and not array.flags.c_contiguous) else 'C' allow_mmap = not self.buffered and not array.dtype.hasobject kwargs = {} try: self.file_handle.tell() except io.UnsupportedOperation: kwargs = {'numpy_array_alignment_bytes': None} wrapper = NumpyArrayWrapper(type(array), array.shape, order, array.dtype, allow_mmap=allow_mmap, **kwargs) return wrapper def save(self, obj): """Subclass the Pickler `save` method. This is a total abuse of the Pickler class in order to use the numpy persistence function `save` instead of the default pickle implementation. The numpy array is replaced by a custom wrapper in the pickle persistence stack and the serialized array is written right after in the file. Warning: the file produced does not follow the pickle format. As such it can not be read with `pickle.load`. """ if self.np is not None and type(obj) in (self.np.ndarray, self.np.matrix, self.np.memmap): if type(obj) is self.np.memmap: # Pickling doesn't work with memmapped arrays obj = self.np.asanyarray(obj) # The array wrapper is pickled instead of the real array. wrapper = self._create_array_wrapper(obj) Pickler.save(self, wrapper) # A framer was introduced with pickle protocol 4 and we want to # ensure the wrapper object is written before the numpy array # buffer in the pickle file. # See https://www.python.org/dev/peps/pep-3154/#framing to get # more information on the framer behavior. if self.proto >= 4: self.framer.commit_frame(force=True) # And then array bytes are written right after the wrapper. wrapper.write_array(obj, self) return return Pickler.save(self, obj) class NumpyUnpickler(Unpickler): """A subclass of the Unpickler to unpickle our numpy pickles. Attributes ---------- mmap_mode: str The memorymap mode to use for reading numpy arrays. file_handle: file_like File object to unpickle from. filename: str Name of the file to unpickle from. It should correspond to file_handle. This parameter is required when using mmap_mode. np: module Reference to numpy module if numpy is installed else None. """ dispatch = Unpickler.dispatch.copy() def __init__(self, filename, file_handle, mmap_mode=None): # The next line is for backward compatibility with pickle generated # with joblib versions less than 0.10. self._dirname = os.path.dirname(filename) self.mmap_mode = mmap_mode self.file_handle = file_handle # filename is required for numpy mmap mode. self.filename = filename self.compat_mode = False Unpickler.__init__(self, self.file_handle) try: import numpy as np except ImportError: np = None self.np = np def load_build(self): """Called to set the state of a newly created object. We capture it to replace our place-holder objects, NDArrayWrapper or NumpyArrayWrapper, by the array we are interested in. We replace them directly in the stack of pickler. NDArrayWrapper is used for backward compatibility with joblib <= 0.9. """ Unpickler.load_build(self) # For backward compatibility, we support NDArrayWrapper objects. if isinstance(self.stack[-1], (NDArrayWrapper, NumpyArrayWrapper)): if self.np is None: raise ImportError("Trying to unpickle an ndarray, " "but numpy didn't import correctly") array_wrapper = self.stack.pop() # If any NDArrayWrapper is found, we switch to compatibility mode, # this will be used to raise a DeprecationWarning to the user at # the end of the unpickling. if isinstance(array_wrapper, NDArrayWrapper): self.compat_mode = True self.stack.append(array_wrapper.read(self)) # Be careful to register our new method. dispatch[pickle.BUILD[0]] = load_build ############################################################################### # Utility functions def dump(value, filename, compress=0, protocol=None, cache_size=None): """Persist an arbitrary Python object into one file. Read more in the :ref:`User Guide `. Parameters ---------- value: any Python object The object to store to disk. filename: str, pathlib.Path, or file object. The file object or path of the file in which it is to be stored. The compression method corresponding to one of the supported filename extensions ('.z', '.gz', '.bz2', '.xz' or '.lzma') will be used automatically. compress: int from 0 to 9 or bool or 2-tuple, optional Optional compression level for the data. 0 or False is no compression. Higher value means more compression, but also slower read and write times. Using a value of 3 is often a good compromise. See the notes for more details. If compress is True, the compression level used is 3. If compress is a 2-tuple, the first element must correspond to a string between supported compressors (e.g 'zlib', 'gzip', 'bz2', 'lzma' 'xz'), the second element must be an integer from 0 to 9, corresponding to the compression level. protocol: int, optional Pickle protocol, see pickle.dump documentation for more details. cache_size: positive int, optional This option is deprecated in 0.10 and has no effect. Returns ------- filenames: list of strings The list of file names in which the data is stored. If compress is false, each array is stored in a different file. See Also -------- joblib.load : corresponding loader Notes ----- Memmapping on load cannot be used for compressed files. Thus using compression can significantly slow down loading. In addition, compressed files take up extra memory during dump and load. """ if Path is not None and isinstance(filename, Path): filename = str(filename) is_filename = isinstance(filename, str) is_fileobj = hasattr(filename, "write") compress_method = 'zlib' # zlib is the default compression method. if compress is True: # By default, if compress is enabled, we want the default compress # level of the compressor. compress_level = None elif isinstance(compress, tuple): # a 2-tuple was set in compress if len(compress) != 2: raise ValueError( 'Compress argument tuple should contain exactly 2 elements: ' '(compress method, compress level), you passed {}' .format(compress)) compress_method, compress_level = compress elif isinstance(compress, str): compress_method = compress compress_level = None # Use default compress level compress = (compress_method, compress_level) else: compress_level = compress if compress_method == 'lz4' and lz4 is None: raise ValueError(LZ4_NOT_INSTALLED_ERROR) if (compress_level is not None and compress_level is not False and compress_level not in range(10)): # Raising an error if a non valid compress level is given. raise ValueError( 'Non valid compress level given: "{}". Possible values are ' '{}.'.format(compress_level, list(range(10)))) if compress_method not in _COMPRESSORS: # Raising an error if an unsupported compression method is given. raise ValueError( 'Non valid compression method given: "{}". Possible values are ' '{}.'.format(compress_method, _COMPRESSORS)) if not is_filename and not is_fileobj: # People keep inverting arguments, and the resulting error is # incomprehensible raise ValueError( 'Second argument should be a filename or a file-like object, ' '%s (type %s) was given.' % (filename, type(filename)) ) if is_filename and not isinstance(compress, tuple): # In case no explicit compression was requested using both compression # method and level in a tuple and the filename has an explicit # extension, we select the corresponding compressor. # unset the variable to be sure no compression level is set afterwards. compress_method = None for name, compressor in _COMPRESSORS.items(): if filename.endswith(compressor.extension): compress_method = name if compress_method in _COMPRESSORS and compress_level == 0: # we choose the default compress_level in case it was not given # as an argument (using compress). compress_level = None if cache_size is not None: # Cache size is deprecated starting from version 0.10 warnings.warn("Please do not set 'cache_size' in joblib.dump, " "this parameter has no effect and will be removed. " "You used 'cache_size={}'".format(cache_size), DeprecationWarning, stacklevel=2) if compress_level != 0: with _write_fileobject(filename, compress=(compress_method, compress_level)) as f: NumpyPickler(f, protocol=protocol).dump(value) elif is_filename: with open(filename, 'wb') as f: NumpyPickler(f, protocol=protocol).dump(value) else: NumpyPickler(filename, protocol=protocol).dump(value) # If the target container is a file object, nothing is returned. if is_fileobj: return # For compatibility, the list of created filenames (e.g with one element # after 0.10.0) is returned by default. return [filename] def _unpickle(fobj, filename="", mmap_mode=None): """Internal unpickling function.""" # We are careful to open the file handle early and keep it open to # avoid race-conditions on renames. # That said, if data is stored in companion files, which can be # the case with the old persistence format, moving the directory # will create a race when joblib tries to access the companion # files. unpickler = NumpyUnpickler(filename, fobj, mmap_mode=mmap_mode) obj = None try: obj = unpickler.load() if unpickler.compat_mode: warnings.warn("The file '%s' has been generated with a " "joblib version less than 0.10. " "Please regenerate this pickle file." % filename, DeprecationWarning, stacklevel=3) except UnicodeDecodeError as exc: # More user-friendly error message new_exc = ValueError( 'You may be trying to read with ' 'python 3 a joblib pickle generated with python 2. ' 'This feature is not supported by joblib.') new_exc.__cause__ = exc raise new_exc return obj def load_temporary_memmap(filename, mmap_mode, unlink_on_gc_collect): from ._memmapping_reducer import JOBLIB_MMAPS, add_maybe_unlink_finalizer obj = load(filename, mmap_mode) JOBLIB_MMAPS.add(obj.filename) if unlink_on_gc_collect: add_maybe_unlink_finalizer(obj) return obj def load(filename, mmap_mode=None): """Reconstruct a Python object from a file persisted with joblib.dump. Read more in the :ref:`User Guide `. WARNING: joblib.load relies on the pickle module and can therefore execute arbitrary Python code. It should therefore never be used to load files from untrusted sources. Parameters ---------- filename: str, pathlib.Path, or file object. The file object or path of the file from which to load the object mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional If not None, the arrays are memory-mapped from the disk. This mode has no effect for compressed files. Note that in this case the reconstructed object might no longer match exactly the originally pickled object. Returns ------- result: any Python object The object stored in the file. See Also -------- joblib.dump : function to save an object Notes ----- This function can load numpy array files saved separately during the dump. If the mmap_mode argument is given, it is passed to np.load and arrays are loaded as memmaps. As a consequence, the reconstructed object might not match the original pickled object. Note that if the file was saved with compression, the arrays cannot be memmapped. """ if Path is not None and isinstance(filename, Path): filename = str(filename) if hasattr(filename, "read"): fobj = filename filename = getattr(fobj, 'name', '') with _read_fileobject(fobj, filename, mmap_mode) as fobj: obj = _unpickle(fobj) else: with open(filename, 'rb') as f: with _read_fileobject(f, filename, mmap_mode) as fobj: if isinstance(fobj, str): # if the returned file object is a string, this means we # try to load a pickle file generated with an version of # Joblib so we load it with joblib compatibility function. return load_compatibility(fobj) obj = _unpickle(fobj, filename, mmap_mode) return obj joblib-1.3.2/joblib/numpy_pickle_compat.py000066400000000000000000000205431446465525000206520ustar00rootroot00000000000000"""Numpy pickle compatibility functions.""" import pickle import os import zlib import inspect from io import BytesIO from .numpy_pickle_utils import _ZFILE_PREFIX from .numpy_pickle_utils import Unpickler from .numpy_pickle_utils import _ensure_native_byte_order def hex_str(an_int): """Convert an int to an hexadecimal string.""" return '{:#x}'.format(an_int) def asbytes(s): if isinstance(s, bytes): return s return s.encode('latin1') _MAX_LEN = len(hex_str(2 ** 64)) _CHUNK_SIZE = 64 * 1024 def read_zfile(file_handle): """Read the z-file and return the content as a string. Z-files are raw data compressed with zlib used internally by joblib for persistence. Backward compatibility is not guaranteed. Do not use for external purposes. """ file_handle.seek(0) header_length = len(_ZFILE_PREFIX) + _MAX_LEN length = file_handle.read(header_length) length = length[len(_ZFILE_PREFIX):] length = int(length, 16) # With python2 and joblib version <= 0.8.4 compressed pickle header is one # character wider so we need to ignore an additional space if present. # Note: the first byte of the zlib data is guaranteed not to be a # space according to # https://tools.ietf.org/html/rfc6713#section-2.1 next_byte = file_handle.read(1) if next_byte != b' ': # The zlib compressed data has started and we need to go back # one byte file_handle.seek(header_length) # We use the known length of the data to tell Zlib the size of the # buffer to allocate. data = zlib.decompress(file_handle.read(), 15, length) assert len(data) == length, ( "Incorrect data length while decompressing %s." "The file could be corrupted." % file_handle) return data def write_zfile(file_handle, data, compress=1): """Write the data in the given file as a Z-file. Z-files are raw data compressed with zlib used internally by joblib for persistence. Backward compatibility is not guaranteed. Do not use for external purposes. """ file_handle.write(_ZFILE_PREFIX) length = hex_str(len(data)) # Store the length of the data file_handle.write(asbytes(length.ljust(_MAX_LEN))) file_handle.write(zlib.compress(asbytes(data), compress)) ############################################################################### # Utility objects for persistence. class NDArrayWrapper(object): """An object to be persisted instead of numpy arrays. The only thing this object does, is to carry the filename in which the array has been persisted, and the array subclass. """ def __init__(self, filename, subclass, allow_mmap=True): """Constructor. Store the useful information for later.""" self.filename = filename self.subclass = subclass self.allow_mmap = allow_mmap def read(self, unpickler): """Reconstruct the array.""" filename = os.path.join(unpickler._dirname, self.filename) # Load the array from the disk # use getattr instead of self.allow_mmap to ensure backward compat # with NDArrayWrapper instances pickled with joblib < 0.9.0 allow_mmap = getattr(self, 'allow_mmap', True) kwargs = {} if allow_mmap: kwargs['mmap_mode'] = unpickler.mmap_mode if "allow_pickle" in inspect.signature(unpickler.np.load).parameters: # Required in numpy 1.16.3 and later to aknowledge the security # risk. kwargs["allow_pickle"] = True array = unpickler.np.load(filename, **kwargs) # Detect byte order mismatch and swap as needed. array = _ensure_native_byte_order(array) # Reconstruct subclasses. This does not work with old # versions of numpy if (hasattr(array, '__array_prepare__') and self.subclass not in (unpickler.np.ndarray, unpickler.np.memmap)): # We need to reconstruct another subclass new_array = unpickler.np.core.multiarray._reconstruct( self.subclass, (0,), 'b') return new_array.__array_prepare__(array) else: return array class ZNDArrayWrapper(NDArrayWrapper): """An object to be persisted instead of numpy arrays. This object store the Zfile filename in which the data array has been persisted, and the meta information to retrieve it. The reason that we store the raw buffer data of the array and the meta information, rather than array representation routine (tobytes) is that it enables us to use completely the strided model to avoid memory copies (a and a.T store as fast). In addition saving the heavy information separately can avoid creating large temporary buffers when unpickling data with large arrays. """ def __init__(self, filename, init_args, state): """Constructor. Store the useful information for later.""" self.filename = filename self.state = state self.init_args = init_args def read(self, unpickler): """Reconstruct the array from the meta-information and the z-file.""" # Here we a simply reproducing the unpickling mechanism for numpy # arrays filename = os.path.join(unpickler._dirname, self.filename) array = unpickler.np.core.multiarray._reconstruct(*self.init_args) with open(filename, 'rb') as f: data = read_zfile(f) state = self.state + (data,) array.__setstate__(state) return array class ZipNumpyUnpickler(Unpickler): """A subclass of the Unpickler to unpickle our numpy pickles.""" dispatch = Unpickler.dispatch.copy() def __init__(self, filename, file_handle, mmap_mode=None): """Constructor.""" self._filename = os.path.basename(filename) self._dirname = os.path.dirname(filename) self.mmap_mode = mmap_mode self.file_handle = self._open_pickle(file_handle) Unpickler.__init__(self, self.file_handle) try: import numpy as np except ImportError: np = None self.np = np def _open_pickle(self, file_handle): return BytesIO(read_zfile(file_handle)) def load_build(self): """Set the state of a newly created object. We capture it to replace our place-holder objects, NDArrayWrapper, by the array we are interested in. We replace them directly in the stack of pickler. """ Unpickler.load_build(self) if isinstance(self.stack[-1], NDArrayWrapper): if self.np is None: raise ImportError("Trying to unpickle an ndarray, " "but numpy didn't import correctly") nd_array_wrapper = self.stack.pop() array = nd_array_wrapper.read(self) self.stack.append(array) dispatch[pickle.BUILD[0]] = load_build def load_compatibility(filename): """Reconstruct a Python object from a file persisted with joblib.dump. This function ensures the compatibility with joblib old persistence format (<= 0.9.3). Parameters ---------- filename: string The name of the file from which to load the object Returns ------- result: any Python object The object stored in the file. See Also -------- joblib.dump : function to save an object Notes ----- This function can load numpy array files saved separately during the dump. """ with open(filename, 'rb') as file_handle: # We are careful to open the file handle early and keep it open to # avoid race-conditions on renames. That said, if data is stored in # companion files, moving the directory will create a race when # joblib tries to access the companion files. unpickler = ZipNumpyUnpickler(filename, file_handle=file_handle) try: obj = unpickler.load() except UnicodeDecodeError as exc: # More user-friendly error message new_exc = ValueError( 'You may be trying to read with ' 'python 3 a joblib pickle generated with python 2. ' 'This feature is not supported by joblib.') new_exc.__cause__ = exc raise new_exc finally: if hasattr(unpickler, 'file_handle'): unpickler.file_handle.close() return obj joblib-1.3.2/joblib/numpy_pickle_utils.py000066400000000000000000000210011446465525000205150ustar00rootroot00000000000000"""Utilities for fast persistence of big data, with optional compression.""" # Author: Gael Varoquaux # Copyright (c) 2009 Gael Varoquaux # License: BSD Style, 3 clauses. import pickle import io import sys import warnings import contextlib from .compressor import _ZFILE_PREFIX from .compressor import _COMPRESSORS try: import numpy as np except ImportError: np = None Unpickler = pickle._Unpickler Pickler = pickle._Pickler xrange = range try: # The python standard library can be built without bz2 so we make bz2 # usage optional. # see https://github.com/scikit-learn/scikit-learn/issues/7526 for more # details. import bz2 except ImportError: bz2 = None # Buffer size used in io.BufferedReader and io.BufferedWriter _IO_BUFFER_SIZE = 1024 ** 2 def _is_raw_file(fileobj): """Check if fileobj is a raw file object, e.g created with open.""" fileobj = getattr(fileobj, 'raw', fileobj) return isinstance(fileobj, io.FileIO) def _get_prefixes_max_len(): # Compute the max prefix len of registered compressors. prefixes = [len(compressor.prefix) for compressor in _COMPRESSORS.values()] prefixes += [len(_ZFILE_PREFIX)] return max(prefixes) def _is_numpy_array_byte_order_mismatch(array): """Check if numpy array is having byte order mismatch""" return ((sys.byteorder == 'big' and (array.dtype.byteorder == '<' or (array.dtype.byteorder == '|' and array.dtype.fields and all(e[0].byteorder == '<' for e in array.dtype.fields.values())))) or (sys.byteorder == 'little' and (array.dtype.byteorder == '>' or (array.dtype.byteorder == '|' and array.dtype.fields and all(e[0].byteorder == '>' for e in array.dtype.fields.values()))))) def _ensure_native_byte_order(array): """Use the byte order of the host while preserving values Does nothing if array already uses the system byte order. """ if _is_numpy_array_byte_order_mismatch(array): array = array.byteswap().newbyteorder('=') return array ############################################################################### # Cache file utilities def _detect_compressor(fileobj): """Return the compressor matching fileobj. Parameters ---------- fileobj: file object Returns ------- str in {'zlib', 'gzip', 'bz2', 'lzma', 'xz', 'compat', 'not-compressed'} """ # Read the magic number in the first bytes of the file. max_prefix_len = _get_prefixes_max_len() if hasattr(fileobj, 'peek'): # Peek allows to read those bytes without moving the cursor in the # file whic. first_bytes = fileobj.peek(max_prefix_len) else: # Fallback to seek if the fileobject is not peekable. first_bytes = fileobj.read(max_prefix_len) fileobj.seek(0) if first_bytes.startswith(_ZFILE_PREFIX): return "compat" else: for name, compressor in _COMPRESSORS.items(): if first_bytes.startswith(compressor.prefix): return name return "not-compressed" def _buffered_read_file(fobj): """Return a buffered version of a read file object.""" return io.BufferedReader(fobj, buffer_size=_IO_BUFFER_SIZE) def _buffered_write_file(fobj): """Return a buffered version of a write file object.""" return io.BufferedWriter(fobj, buffer_size=_IO_BUFFER_SIZE) @contextlib.contextmanager def _read_fileobject(fileobj, filename, mmap_mode=None): """Utility function opening the right fileobject from a filename. The magic number is used to choose between the type of file object to open: * regular file object (default) * zlib file object * gzip file object * bz2 file object * lzma file object (for xz and lzma compressor) Parameters ---------- fileobj: file object compressor: str in {'zlib', 'gzip', 'bz2', 'lzma', 'xz', 'compat', 'not-compressed'} filename: str filename path corresponding to the fileobj parameter. mmap_mode: str memory map mode that should be used to open the pickle file. This parameter is useful to verify that the user is not trying to one with compression. Default: None. Returns ------- a file like object """ # Detect if the fileobj contains compressed data. compressor = _detect_compressor(fileobj) if compressor == 'compat': # Compatibility with old pickle mode: simply return the input # filename "as-is" and let the compatibility function be called by the # caller. warnings.warn("The file '%s' has been generated with a joblib " "version less than 0.10. " "Please regenerate this pickle file." % filename, DeprecationWarning, stacklevel=2) yield filename else: if compressor in _COMPRESSORS: # based on the compressor detected in the file, we open the # correct decompressor file object, wrapped in a buffer. compressor_wrapper = _COMPRESSORS[compressor] inst = compressor_wrapper.decompressor_file(fileobj) fileobj = _buffered_read_file(inst) # Checking if incompatible load parameters with the type of file: # mmap_mode cannot be used with compressed file or in memory buffers # such as io.BytesIO. if mmap_mode is not None: if isinstance(fileobj, io.BytesIO): warnings.warn('In memory persistence is not compatible with ' 'mmap_mode "%(mmap_mode)s" flag passed. ' 'mmap_mode option will be ignored.' % locals(), stacklevel=2) elif compressor != 'not-compressed': warnings.warn('mmap_mode "%(mmap_mode)s" is not compatible ' 'with compressed file %(filename)s. ' '"%(mmap_mode)s" flag will be ignored.' % locals(), stacklevel=2) elif not _is_raw_file(fileobj): warnings.warn('"%(fileobj)r" is not a raw file, mmap_mode ' '"%(mmap_mode)s" flag will be ignored.' % locals(), stacklevel=2) yield fileobj def _write_fileobject(filename, compress=("zlib", 3)): """Return the right compressor file object in write mode.""" compressmethod = compress[0] compresslevel = compress[1] if compressmethod in _COMPRESSORS.keys(): file_instance = _COMPRESSORS[compressmethod].compressor_file( filename, compresslevel=compresslevel) return _buffered_write_file(file_instance) else: file_instance = _COMPRESSORS['zlib'].compressor_file( filename, compresslevel=compresslevel) return _buffered_write_file(file_instance) # Utility functions/variables from numpy required for writing arrays. # We need at least the functions introduced in version 1.9 of numpy. Here, # we use the ones from numpy 1.10.2. BUFFER_SIZE = 2 ** 18 # size of buffer for reading npz files in bytes def _read_bytes(fp, size, error_template="ran out of data"): """Read from file-like object until size bytes are read. TODO python2_drop: is it still needed? The docstring mentions python 2.6 and it looks like this can be at least simplified ... Raises ValueError if not EOF is encountered before size bytes are read. Non-blocking objects only supported if they derive from io objects. Required as e.g. ZipExtFile in python 2.6 can return less data than requested. This function was taken from numpy/lib/format.py in version 1.10.2. Parameters ---------- fp: file-like object size: int error_template: str Returns ------- a bytes object The data read in bytes. """ data = bytes() while True: # io files (default in python3) return None or raise on # would-block, python2 file will truncate, probably nothing can be # done about that. note that regular files can't be non-blocking try: r = fp.read(size - len(data)) data += r if len(r) == 0 or len(data) == size: break except io.BlockingIOError: pass if len(data) != size: msg = "EOF: reading %s, expected %d bytes got %d" raise ValueError(msg % (error_template, size, len(data))) else: return data joblib-1.3.2/joblib/parallel.py000066400000000000000000002367451446465525000164210ustar00rootroot00000000000000""" Helpers for embarrassingly parallel code. """ # Author: Gael Varoquaux < gael dot varoquaux at normalesup dot org > # Copyright: 2010, Gael Varoquaux # License: BSD 3 clause from __future__ import division import os import sys from math import sqrt import functools import collections import time import threading import itertools from uuid import uuid4 from numbers import Integral import warnings import queue import weakref from contextlib import nullcontext from multiprocessing import TimeoutError from ._multiprocessing_helpers import mp from .logger import Logger, short_format_time from .disk import memstr_to_bytes from ._parallel_backends import (FallbackToBackend, MultiprocessingBackend, ThreadingBackend, SequentialBackend, LokyBackend) from ._utils import eval_expr, _Sentinel # Make sure that those two classes are part of the public joblib.parallel API # so that 3rd party backend implementers can import them from here. from ._parallel_backends import AutoBatchingMixin # noqa from ._parallel_backends import ParallelBackendBase # noqa IS_PYPY = hasattr(sys, "pypy_version_info") BACKENDS = { 'threading': ThreadingBackend, 'sequential': SequentialBackend, } # name of the backend used by default by Parallel outside of any context # managed by ``parallel_config`` or ``parallel_backend``. # threading is the only backend that is always everywhere DEFAULT_BACKEND = 'threading' MAYBE_AVAILABLE_BACKENDS = {'multiprocessing', 'loky'} # if multiprocessing is available, so is loky, we set it as the default # backend if mp is not None: BACKENDS['multiprocessing'] = MultiprocessingBackend from .externals import loky BACKENDS['loky'] = LokyBackend DEFAULT_BACKEND = 'loky' DEFAULT_THREAD_BACKEND = 'threading' # Thread local value that can be overridden by the ``parallel_config`` context # manager _backend = threading.local() def _register_dask(): """Register Dask Backend if called with parallel_config(backend="dask")""" try: from ._dask import DaskDistributedBackend register_parallel_backend('dask', DaskDistributedBackend) except ImportError as e: msg = ("To use the dask.distributed backend you must install both " "the `dask` and distributed modules.\n\n" "See https://dask.pydata.org/en/latest/install.html for more " "information.") raise ImportError(msg) from e EXTERNAL_BACKENDS = { 'dask': _register_dask, } # Sentinels for the default values of the Parallel constructor and # the parallel_config and parallel_backend context managers default_parallel_config = { "backend": _Sentinel(default_value=None), "n_jobs": _Sentinel(default_value=None), "verbose": _Sentinel(default_value=0), "temp_folder": _Sentinel(default_value=None), "max_nbytes": _Sentinel(default_value="1M"), "mmap_mode": _Sentinel(default_value="r"), "prefer": _Sentinel(default_value=None), "require": _Sentinel(default_value=None), } VALID_BACKEND_HINTS = ('processes', 'threads', None) VALID_BACKEND_CONSTRAINTS = ('sharedmem', None) def _get_config_param(param, context_config, key): """Return the value of a parallel config parameter Explicitly setting it in Parallel has priority over setting in a parallel_(config/backend) context manager. """ if param is not default_parallel_config[key]: # param is explicitely set, return it return param if context_config[key] is not default_parallel_config[key]: # there's a context manager and the key is set, return it return context_config[key] # Otherwise, we are in the default_parallel_config, # return the default value return param.default_value def get_active_backend( prefer=default_parallel_config["prefer"], require=default_parallel_config["require"], verbose=default_parallel_config["verbose"], ): """Return the active default backend""" backend, config = _get_active_backend(prefer, require, verbose) n_jobs = _get_config_param( default_parallel_config['n_jobs'], config, "n_jobs" ) return backend, n_jobs def _get_active_backend( prefer=default_parallel_config["prefer"], require=default_parallel_config["require"], verbose=default_parallel_config["verbose"], ): """Return the active default backend""" backend_config = getattr(_backend, "config", default_parallel_config) backend = _get_config_param( default_parallel_config['backend'], backend_config, "backend" ) prefer = _get_config_param(prefer, backend_config, "prefer") require = _get_config_param(require, backend_config, "require") verbose = _get_config_param(verbose, backend_config, "verbose") if prefer not in VALID_BACKEND_HINTS: raise ValueError( f"prefer={prefer} is not a valid backend hint, " f"expected one of {VALID_BACKEND_HINTS}" ) if require not in VALID_BACKEND_CONSTRAINTS: raise ValueError( f"require={require} is not a valid backend constraint, " f"expected one of {VALID_BACKEND_CONSTRAINTS}" ) if prefer == 'processes' and require == 'sharedmem': raise ValueError( "prefer == 'processes' and require == 'sharedmem'" " are inconsistent settings" ) explicit_backend = True if backend is None: # We are either outside of the scope of any parallel_(config/backend) # context manager or the context manager did not set a backend. # create the default backend instance now. backend = BACKENDS[DEFAULT_BACKEND](nesting_level=0) explicit_backend = False # Try to use the backend set by the user with the context manager. nesting_level = backend.nesting_level uses_threads = getattr(backend, 'uses_threads', False) supports_sharedmem = getattr(backend, 'supports_sharedmem', False) # Force to use thread-based backend if the provided backend does not # match the shared memory constraint or if the backend is not explicitely # given and threads are prefered. force_threads = (require == 'sharedmem' and not supports_sharedmem) force_threads |= ( not explicit_backend and prefer == 'threads' and not uses_threads ) if force_threads: # This backend does not match the shared memory constraint: # fallback to the default thead-based backend. sharedmem_backend = BACKENDS[DEFAULT_THREAD_BACKEND]( nesting_level=nesting_level ) # Warn the user if we forced the backend to thread-based, while the # user explicitely specified a non-thread-based backend. if verbose >= 10 and explicit_backend: print( f"Using {sharedmem_backend.__class__.__name__} as " f"joblib backend instead of {backend.__class__.__name__} " "as the latter does not provide shared memory semantics." ) # Force to n_jobs=1 by default thread_config = backend_config.copy() thread_config['n_jobs'] = 1 return sharedmem_backend, thread_config return backend, backend_config class parallel_config: """Set the default backend or configuration for :class:`~joblib.Parallel`. This is an alternative to directly passing keyword arguments to the :class:`~joblib.Parallel` class constructor. It is particularly useful when calling into library code that uses joblib internally but does not expose the various parallel configuration arguments in its own API. Parameters ---------- backend : str or ParallelBackendBase instance, default=None If ``backend`` is a string it must match a previously registered implementation using the :func:`~register_parallel_backend` function. By default the following backends are available: - 'loky': single-host, process-based parallelism (used by default), - 'threading': single-host, thread-based parallelism, - 'multiprocessing': legacy single-host, process-based parallelism. 'loky' is recommended to run functions that manipulate Python objects. 'threading' is a low-overhead alternative that is most efficient for functions that release the Global Interpreter Lock: e.g. I/O-bound code or CPU-bound code in a few calls to native code that explicitly releases the GIL. Note that on some rare systems (such as pyodide), multiprocessing and loky may not be available, in which case joblib defaults to threading. In addition, if the ``dask`` and ``distributed`` Python packages are installed, it is possible to use the 'dask' backend for better scheduling of nested parallel calls without over-subscription and potentially distribute parallel calls over a networked cluster of several hosts. It is also possible to use the distributed 'ray' backend for distributing the workload to a cluster of nodes. See more details in the Examples section below. Alternatively the backend can be passed directly as an instance. n_jobs : int, default=None The maximum number of concurrently running jobs, such as the number of Python worker processes when ``backend="loky"`` or the size of the thread-pool when ``backend="threading"``. If -1 all CPUs are used. If 1 is given, no parallel computing code is used at all, which is useful for debugging. For ``n_jobs`` below -1, (n_cpus + 1 + n_jobs) are used. Thus for ``n_jobs=-2``, all CPUs but one are used. ``None`` is a marker for 'unset' that will be interpreted as ``n_jobs=1`` in most backends. verbose : int, default=0 The verbosity level: if non zero, progress messages are printed. Above 50, the output is sent to stdout. The frequency of the messages increases with the verbosity level. If it more than 10, all iterations are reported. temp_folder : str, default=None Folder to be used by the pool for memmapping large arrays for sharing memory with worker processes. If None, this will try in order: - a folder pointed by the ``JOBLIB_TEMP_FOLDER`` environment variable, - ``/dev/shm`` if the folder exists and is writable: this is a RAM disk filesystem available by default on modern Linux distributions, - the default system temporary folder that can be overridden with ``TMP``, ``TMPDIR`` or ``TEMP`` environment variables, typically ``/tmp`` under Unix operating systems. max_nbytes int, str, or None, optional, default='1M' Threshold on the size of arrays passed to the workers that triggers automated memory mapping in temp_folder. Can be an int in Bytes, or a human-readable string, e.g., '1M' for 1 megabyte. Use None to disable memmapping of large arrays. mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, default='r' Memmapping mode for numpy arrays passed to workers. None will disable memmapping, other modes defined in the numpy.memmap doc: https://numpy.org/doc/stable/reference/generated/numpy.memmap.html Also, see 'max_nbytes' parameter documentation for more details. prefer: str in {'processes', 'threads'} or None, default=None Soft hint to choose the default backend. The default process-based backend is 'loky' and the default thread-based backend is 'threading'. Ignored if the ``backend`` parameter is specified. require: 'sharedmem' or None, default=None Hard constraint to select the backend. If set to 'sharedmem', the selected backend will be single-host and thread-based. inner_max_num_threads : int, default=None If not None, overwrites the limit set on the number of threads usable in some third-party library threadpools like OpenBLAS, MKL or OpenMP. This is only used with the ``loky`` backend. backend_params : dict Additional parameters to pass to the backend constructor when backend is a string. Notes ----- Joblib tries to limit the oversubscription by limiting the number of threads usable in some third-party library threadpools like OpenBLAS, MKL or OpenMP. The default limit in each worker is set to ``max(cpu_count() // effective_n_jobs, 1)`` but this limit can be overwritten with the ``inner_max_num_threads`` argument which will be used to set this limit in the child processes. .. versionadded:: 1.3 Examples -------- >>> from operator import neg >>> with parallel_config(backend='threading'): ... print(Parallel()(delayed(neg)(i + 1) for i in range(5))) ... [-1, -2, -3, -4, -5] To use the 'ray' joblib backend add the following lines: >>> from ray.util.joblib import register_ray # doctest: +SKIP >>> register_ray() # doctest: +SKIP >>> with parallel_config(backend="ray"): # doctest: +SKIP ... print(Parallel()(delayed(neg)(i + 1) for i in range(5))) [-1, -2, -3, -4, -5] """ def __init__( self, backend=default_parallel_config["backend"], *, n_jobs=default_parallel_config["n_jobs"], verbose=default_parallel_config["verbose"], temp_folder=default_parallel_config["temp_folder"], max_nbytes=default_parallel_config["max_nbytes"], mmap_mode=default_parallel_config["mmap_mode"], prefer=default_parallel_config["prefer"], require=default_parallel_config["require"], inner_max_num_threads=None, **backend_params ): # Save the parallel info and set the active parallel config self.old_parallel_config = getattr( _backend, "config", default_parallel_config ) backend = self._check_backend( backend, inner_max_num_threads, **backend_params ) new_config = { "n_jobs": n_jobs, "verbose": verbose, "temp_folder": temp_folder, "max_nbytes": max_nbytes, "mmap_mode": mmap_mode, "prefer": prefer, "require": require, "backend": backend } self.parallel_config = self.old_parallel_config.copy() self.parallel_config.update({ k: v for k, v in new_config.items() if not isinstance(v, _Sentinel) }) setattr(_backend, "config", self.parallel_config) def _check_backend(self, backend, inner_max_num_threads, **backend_params): if backend is default_parallel_config['backend']: if inner_max_num_threads is not None or len(backend_params) > 0: raise ValueError( "inner_max_num_threads and other constructor " "parameters backend_params are only supported " "when backend is not None." ) return backend if isinstance(backend, str): # Handle non-registered or missing backends if backend not in BACKENDS: if backend in EXTERNAL_BACKENDS: register = EXTERNAL_BACKENDS[backend] register() elif backend in MAYBE_AVAILABLE_BACKENDS: warnings.warn( f"joblib backend '{backend}' is not available on " f"your system, falling back to {DEFAULT_BACKEND}.", UserWarning, stacklevel=2 ) BACKENDS[backend] = BACKENDS[DEFAULT_BACKEND] else: raise ValueError( f"Invalid backend: {backend}, expected one of " f"{sorted(BACKENDS.keys())}" ) backend = BACKENDS[backend](**backend_params) if inner_max_num_threads is not None: msg = ( f"{backend.__class__.__name__} does not accept setting the " "inner_max_num_threads argument." ) assert backend.supports_inner_max_num_threads, msg backend.inner_max_num_threads = inner_max_num_threads # If the nesting_level of the backend is not set previously, use the # nesting level from the previous active_backend to set it if backend.nesting_level is None: parent_backend = self.old_parallel_config['backend'] if parent_backend is default_parallel_config['backend']: nesting_level = 0 else: nesting_level = parent_backend.nesting_level backend.nesting_level = nesting_level return backend def __enter__(self): return self.parallel_config def __exit__(self, type, value, traceback): self.unregister() def unregister(self): setattr(_backend, "config", self.old_parallel_config) class parallel_backend(parallel_config): """Change the default backend used by Parallel inside a with block. .. warning:: It is advised to use the :class:`~joblib.parallel_config` context manager instead, which allows more fine-grained control over the backend configuration. If ``backend`` is a string it must match a previously registered implementation using the :func:`~register_parallel_backend` function. By default the following backends are available: - 'loky': single-host, process-based parallelism (used by default), - 'threading': single-host, thread-based parallelism, - 'multiprocessing': legacy single-host, process-based parallelism. 'loky' is recommended to run functions that manipulate Python objects. 'threading' is a low-overhead alternative that is most efficient for functions that release the Global Interpreter Lock: e.g. I/O-bound code or CPU-bound code in a few calls to native code that explicitly releases the GIL. Note that on some rare systems (such as Pyodide), multiprocessing and loky may not be available, in which case joblib defaults to threading. You can also use the `Dask `_ joblib backend to distribute work across machines. This works well with scikit-learn estimators with the ``n_jobs`` parameter, for example:: >>> import joblib # doctest: +SKIP >>> from sklearn.model_selection import GridSearchCV # doctest: +SKIP >>> from dask.distributed import Client, LocalCluster # doctest: +SKIP >>> # create a local Dask cluster >>> cluster = LocalCluster() # doctest: +SKIP >>> client = Client(cluster) # doctest: +SKIP >>> grid_search = GridSearchCV(estimator, param_grid, n_jobs=-1) ... # doctest: +SKIP >>> with joblib.parallel_backend("dask", scatter=[X, y]): # doctest: +SKIP ... grid_search.fit(X, y) It is also possible to use the distributed 'ray' backend for distributing the workload to a cluster of nodes. To use the 'ray' joblib backend add the following lines:: >>> from ray.util.joblib import register_ray # doctest: +SKIP >>> register_ray() # doctest: +SKIP >>> with parallel_backend("ray"): # doctest: +SKIP ... print(Parallel()(delayed(neg)(i + 1) for i in range(5))) [-1, -2, -3, -4, -5] Alternatively the backend can be passed directly as an instance. By default all available workers will be used (``n_jobs=-1``) unless the caller passes an explicit value for the ``n_jobs`` parameter. This is an alternative to passing a ``backend='backend_name'`` argument to the :class:`~Parallel` class constructor. It is particularly useful when calling into library code that uses joblib internally but does not expose the backend argument in its own API. >>> from operator import neg >>> with parallel_backend('threading'): ... print(Parallel()(delayed(neg)(i + 1) for i in range(5))) ... [-1, -2, -3, -4, -5] Joblib also tries to limit the oversubscription by limiting the number of threads usable in some third-party library threadpools like OpenBLAS, MKL or OpenMP. The default limit in each worker is set to ``max(cpu_count() // effective_n_jobs, 1)`` but this limit can be overwritten with the ``inner_max_num_threads`` argument which will be used to set this limit in the child processes. .. versionadded:: 0.10 See Also -------- joblib.parallel_config : context manager to change the backend configuration. """ def __init__(self, backend, n_jobs=-1, inner_max_num_threads=None, **backend_params): super().__init__( backend=backend, n_jobs=n_jobs, inner_max_num_threads=inner_max_num_threads, **backend_params ) if self.old_parallel_config is None: self.old_backend_and_jobs = None else: self.old_backend_and_jobs = ( self.old_parallel_config["backend"], self.old_parallel_config["n_jobs"], ) self.new_backend_and_jobs = ( self.parallel_config["backend"], self.parallel_config["n_jobs"], ) def __enter__(self): return self.new_backend_and_jobs # Under Linux or OS X the default start method of multiprocessing # can cause third party libraries to crash. Under Python 3.4+ it is possible # to set an environment variable to switch the default start method from # 'fork' to 'forkserver' or 'spawn' to avoid this issue albeit at the cost # of causing semantic changes and some additional pool instantiation overhead. DEFAULT_MP_CONTEXT = None if hasattr(mp, 'get_context'): method = os.environ.get('JOBLIB_START_METHOD', '').strip() or None if method is not None: DEFAULT_MP_CONTEXT = mp.get_context(method=method) class BatchedCalls(object): """Wrap a sequence of (func, args, kwargs) tuples as a single callable""" def __init__(self, iterator_slice, backend_and_jobs, reducer_callback=None, pickle_cache=None): self.items = list(iterator_slice) self._size = len(self.items) self._reducer_callback = reducer_callback if isinstance(backend_and_jobs, tuple): self._backend, self._n_jobs = backend_and_jobs else: # this is for backward compatibility purposes. Before 0.12.6, # nested backends were returned without n_jobs indications. self._backend, self._n_jobs = backend_and_jobs, None self._pickle_cache = pickle_cache if pickle_cache is not None else {} def __call__(self): # Set the default nested backend to self._backend but do not set the # change the default number of processes to -1 with parallel_config(backend=self._backend, n_jobs=self._n_jobs): return [func(*args, **kwargs) for func, args, kwargs in self.items] def __reduce__(self): if self._reducer_callback is not None: self._reducer_callback() # no need to pickle the callback. return ( BatchedCalls, (self.items, (self._backend, self._n_jobs), None, self._pickle_cache) ) def __len__(self): return self._size # Possible exit status for a task TASK_DONE = "Done" TASK_ERROR = "Error" TASK_PENDING = "Pending" ############################################################################### # CPU count that works also when multiprocessing has been disabled via # the JOBLIB_MULTIPROCESSING environment variable def cpu_count(only_physical_cores=False): """Return the number of CPUs. This delegates to loky.cpu_count that takes into account additional constraints such as Linux CFS scheduler quotas (typically set by container runtimes such as docker) and CPU affinity (for instance using the taskset command on Linux). If only_physical_cores is True, do not take hyperthreading / SMT logical cores into account. """ if mp is None: return 1 return loky.cpu_count(only_physical_cores=only_physical_cores) ############################################################################### # For verbosity def _verbosity_filter(index, verbose): """ Returns False for indices increasingly apart, the distance depending on the value of verbose. We use a lag increasing as the square of index """ if not verbose: return True elif verbose > 10: return False if index == 0: return False verbose = .5 * (11 - verbose) ** 2 scale = sqrt(index / verbose) next_scale = sqrt((index + 1) / verbose) return (int(next_scale) == int(scale)) ############################################################################### def delayed(function): """Decorator used to capture the arguments of a function.""" def delayed_function(*args, **kwargs): return function, args, kwargs try: delayed_function = functools.wraps(function)(delayed_function) except AttributeError: " functools.wraps fails on some callable objects " return delayed_function ############################################################################### class BatchCompletionCallBack(object): """Callback to keep track of completed results and schedule the next tasks. This callable is executed by the parent process whenever a worker process has completed a batch of tasks. It is used for progress reporting, to update estimate of the batch processing duration and to schedule the next batch of tasks to be processed. It is assumed that this callback will always be triggered by the backend right after the end of a task, in case of success as well as in case of failure. """ ########################################################################## # METHODS CALLED BY THE MAIN THREAD # ########################################################################## def __init__(self, dispatch_timestamp, batch_size, parallel): self.dispatch_timestamp = dispatch_timestamp self.batch_size = batch_size self.parallel = parallel self.parallel_call_id = parallel._call_id # Internals to keep track of the status and outcome of the task. # Used to hold a reference to the future-like object returned by the # backend after launching this task # This will be set later when calling `register_job`, as it is only # created once the task has been submitted. self.job = None if not parallel._backend.supports_retrieve_callback: # The status is only used for asynchronous result retrieval in the # callback. self.status = None else: # The initial status for the job is TASK_PENDING. # Once it is done, it will be either TASK_DONE, or TASK_ERROR. self.status = TASK_PENDING def register_job(self, job): """Register the object returned by `apply_async`.""" self.job = job def get_result(self, timeout): """Returns the raw result of the task that was submitted. If the task raised an exception rather than returning, this same exception will be raised instead. If the backend supports the retrieval callback, it is assumed that this method is only called after the result has been registered. It is ensured by checking that `self.status(timeout)` does not return TASK_PENDING. In this case, `get_result` directly returns the registered result (or raise the registered exception). For other backends, there are no such assumptions, but `get_result` still needs to synchronously retrieve the result before it can return it or raise. It will block at most `self.timeout` seconds waiting for retrieval to complete, after that it raises a TimeoutError. """ backend = self.parallel._backend if backend.supports_retrieve_callback: # We assume that the result has already been retrieved by the # callback thread, and is stored internally. It's just waiting to # be returned. return self._return_or_raise() # For other backends, the main thread needs to run the retrieval step. try: if backend.supports_timeout: result = self.job.get(timeout=timeout) else: result = self.job.get() outcome = dict(result=result, status=TASK_DONE) except BaseException as e: outcome = dict(result=e, status=TASK_ERROR) self._register_outcome(outcome) return self._return_or_raise() def _return_or_raise(self): try: if self.status == TASK_ERROR: raise self._result return self._result finally: del self._result def get_status(self, timeout): """Get the status of the task. This function also checks if the timeout has been reached and register the TimeoutError outcome when it is the case. """ if timeout is None or self.status != TASK_PENDING: return self.status # The computation are running and the status is pending. # Check that we did not wait for this jobs more than `timeout`. now = time.time() if not hasattr(self, "_completion_timeout_counter"): self._completion_timeout_counter = now if (now - self._completion_timeout_counter) > timeout: outcome = dict(result=TimeoutError(), status=TASK_ERROR) self._register_outcome(outcome) return self.status ########################################################################## # METHODS CALLED BY CALLBACK THREADS # ########################################################################## def __call__(self, out): """Function called by the callback thread after a job is completed.""" # If the backend doesn't support callback retrievals, the next batch of # tasks is dispatched regardless. The result will be retrieved by the # main thread when calling `get_result`. if not self.parallel._backend.supports_retrieve_callback: self._dispatch_new() return # If the backend supports retrieving the result in the callback, it # registers the task outcome (TASK_ERROR or TASK_DONE), and schedules # the next batch if needed. with self.parallel._lock: # Edge case where while the task was processing, the `parallel` # instance has been reset and a new call has been issued, but the # worker managed to complete the task and trigger this callback # call just before being aborted by the reset. if self.parallel._call_id != self.parallel_call_id: return # When aborting, stop as fast as possible and do not retrieve the # result as it won't be returned by the Parallel call. if self.parallel._aborting: return # Retrieves the result of the task in the main process and dispatch # a new batch if needed. job_succeeded = self._retrieve_result(out) if job_succeeded: self._dispatch_new() def _dispatch_new(self): """Schedule the next batch of tasks to be processed.""" # This steps ensure that auto-baching works as expected. this_batch_duration = time.time() - self.dispatch_timestamp self.parallel._backend.batch_completed(self.batch_size, this_batch_duration) # Schedule the next batch of tasks. with self.parallel._lock: self.parallel.n_completed_tasks += self.batch_size self.parallel.print_progress() if self.parallel._original_iterator is not None: self.parallel.dispatch_next() def _retrieve_result(self, out): """Fetch and register the outcome of a task. Return True if the task succeeded, False otherwise. This function is only called by backends that support retrieving the task result in the callback thread. """ try: result = self.parallel._backend.retrieve_result_callback(out) outcome = dict(status=TASK_DONE, result=result) except BaseException as e: # Avoid keeping references to parallel in the error. e.__traceback__ = None outcome = dict(result=e, status=TASK_ERROR) self._register_outcome(outcome) return outcome['status'] != TASK_ERROR ########################################################################## # This method can be called either in the main thread # # or in the callback thread. # ########################################################################## def _register_outcome(self, outcome): """Register the outcome of a task. This method can be called only once, future calls will be ignored. """ # Covers the edge case where the main thread tries to register a # `TimeoutError` while the callback thread tries to register a result # at the same time. with self.parallel._lock: if self.status not in (TASK_PENDING, None): return self.status = outcome["status"] self._result = outcome["result"] # Once the result and the status are extracted, the last reference to # the job can be deleted. self.job = None # As soon as an error as been spotted, early stopping flags are sent to # the `parallel` instance. if self.status == TASK_ERROR: self.parallel._exception = True self.parallel._aborting = True ############################################################################### def register_parallel_backend(name, factory, make_default=False): """Register a new Parallel backend factory. The new backend can then be selected by passing its name as the backend argument to the :class:`~Parallel` class. Moreover, the default backend can be overwritten globally by setting make_default=True. The factory can be any callable that takes no argument and return an instance of ``ParallelBackendBase``. Warning: this function is experimental and subject to change in a future version of joblib. .. versionadded:: 0.10 """ BACKENDS[name] = factory if make_default: global DEFAULT_BACKEND DEFAULT_BACKEND = name def effective_n_jobs(n_jobs=-1): """Determine the number of jobs that can actually run in parallel n_jobs is the number of workers requested by the callers. Passing n_jobs=-1 means requesting all available workers for instance matching the number of CPU cores on the worker host(s). This method should return a guesstimate of the number of workers that can actually perform work concurrently with the currently enabled default backend. The primary use case is to make it possible for the caller to know in how many chunks to slice the work. In general working on larger data chunks is more efficient (less scheduling overhead and better use of CPU cache prefetching heuristics) as long as all the workers have enough work to do. Warning: this function is experimental and subject to change in a future version of joblib. .. versionadded:: 0.10 """ if n_jobs == 1: return 1 backend, backend_n_jobs = get_active_backend() if n_jobs is None: n_jobs = backend_n_jobs return backend.effective_n_jobs(n_jobs=n_jobs) ############################################################################### class Parallel(Logger): ''' Helper class for readable parallel mapping. Read more in the :ref:`User Guide `. Parameters ---------- n_jobs: int, default: None The maximum number of concurrently running jobs, such as the number of Python worker processes when backend="multiprocessing" or the size of the thread-pool when backend="threading". If -1 all CPUs are used. If 1 is given, no parallel computing code is used at all, and the behavior amounts to a simple python `for` loop. This mode is not compatible with `timeout`. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used. Thus for n_jobs = -2, all CPUs but one are used. None is a marker for 'unset' that will be interpreted as n_jobs=1 unless the call is performed under a :func:`~parallel_config` context manager that sets another value for ``n_jobs``. backend: str, ParallelBackendBase instance or None, default: 'loky' Specify the parallelization backend implementation. Supported backends are: - "loky" used by default, can induce some communication and memory overhead when exchanging input and output data with the worker Python processes. On some rare systems (such as Pyiodide), the loky backend may not be available. - "multiprocessing" previous process-based backend based on `multiprocessing.Pool`. Less robust than `loky`. - "threading" is a very low-overhead backend but it suffers from the Python Global Interpreter Lock if the called function relies a lot on Python objects. "threading" is mostly useful when the execution bottleneck is a compiled extension that explicitly releases the GIL (for instance a Cython loop wrapped in a "with nogil" block or an expensive call to a library such as NumPy). - finally, you can register backends by calling :func:`~register_parallel_backend`. This will allow you to implement a backend of your liking. It is not recommended to hard-code the backend name in a call to :class:`~Parallel` in a library. Instead it is recommended to set soft hints (prefer) or hard constraints (require) so as to make it possible for library users to change the backend from the outside using the :func:`~parallel_config` context manager. return_as: str in {'list', 'generator'}, default: 'list' If 'list', calls to this instance will return a list, only when all results have been processed and retrieved. If 'generator', it will return a generator that yields the results as soon as they are available, in the order the tasks have been submitted with. Future releases are planned to also support 'generator_unordered', in which case the generator immediately yields available results independently of the submission order. prefer: str in {'processes', 'threads'} or None, default: None Soft hint to choose the default backend if no specific backend was selected with the :func:`~parallel_config` context manager. The default process-based backend is 'loky' and the default thread-based backend is 'threading'. Ignored if the ``backend`` parameter is specified. require: 'sharedmem' or None, default None Hard constraint to select the backend. If set to 'sharedmem', the selected backend will be single-host and thread-based even if the user asked for a non-thread based backend with :func:`~joblib.parallel_config`. verbose: int, optional The verbosity level: if non zero, progress messages are printed. Above 50, the output is sent to stdout. The frequency of the messages increases with the verbosity level. If it more than 10, all iterations are reported. timeout: float, optional Timeout limit for each task to complete. If any task takes longer a TimeOutError will be raised. Only applied when n_jobs != 1 pre_dispatch: {'all', integer, or expression, as in '3*n_jobs'} The number of batches (of tasks) to be pre-dispatched. Default is '2*n_jobs'. When batch_size="auto" this is reasonable default and the workers should never starve. Note that only basic arithmetics are allowed here and no modules can be used in this expression. batch_size: int or 'auto', default: 'auto' The number of atomic tasks to dispatch at once to each worker. When individual evaluations are very fast, dispatching calls to workers can be slower than sequential computation because of the overhead. Batching fast computations together can mitigate this. The ``'auto'`` strategy keeps track of the time it takes for a batch to complete, and dynamically adjusts the batch size to keep the time on the order of half a second, using a heuristic. The initial batch size is 1. ``batch_size="auto"`` with ``backend="threading"`` will dispatch batches of a single task at a time as the threading backend has very little overhead and using larger batch size has not proved to bring any gain in that case. temp_folder: str, optional Folder to be used by the pool for memmapping large arrays for sharing memory with worker processes. If None, this will try in order: - a folder pointed by the JOBLIB_TEMP_FOLDER environment variable, - /dev/shm if the folder exists and is writable: this is a RAM disk filesystem available by default on modern Linux distributions, - the default system temporary folder that can be overridden with TMP, TMPDIR or TEMP environment variables, typically /tmp under Unix operating systems. Only active when backend="loky" or "multiprocessing". max_nbytes int, str, or None, optional, 1M by default Threshold on the size of arrays passed to the workers that triggers automated memory mapping in temp_folder. Can be an int in Bytes, or a human-readable string, e.g., '1M' for 1 megabyte. Use None to disable memmapping of large arrays. Only active when backend="loky" or "multiprocessing". mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, default: 'r' Memmapping mode for numpy arrays passed to workers. None will disable memmapping, other modes defined in the numpy.memmap doc: https://numpy.org/doc/stable/reference/generated/numpy.memmap.html Also, see 'max_nbytes' parameter documentation for more details. Notes ----- This object uses workers to compute in parallel the application of a function to many different arguments. The main functionality it brings in addition to using the raw multiprocessing or concurrent.futures API are (see examples for details): * More readable code, in particular since it avoids constructing list of arguments. * Easier debugging: - informative tracebacks even when the error happens on the client side - using 'n_jobs=1' enables to turn off parallel computing for debugging without changing the codepath - early capture of pickling errors * An optional progress meter. * Interruption of multiprocesses jobs with 'Ctrl-C' * Flexible pickling control for the communication to and from the worker processes. * Ability to use shared memory efficiently with worker processes for large numpy-based datastructures. Note that the intended usage is to run one call at a time. Multiple calls to the same Parallel object will result in a ``RuntimeError`` Examples -------- A simple example: >>> from math import sqrt >>> from joblib import Parallel, delayed >>> Parallel(n_jobs=1)(delayed(sqrt)(i**2) for i in range(10)) [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] Reshaping the output when the function has several return values: >>> from math import modf >>> from joblib import Parallel, delayed >>> r = Parallel(n_jobs=1)(delayed(modf)(i/2.) for i in range(10)) >>> res, i = zip(*r) >>> res (0.0, 0.5, 0.0, 0.5, 0.0, 0.5, 0.0, 0.5, 0.0, 0.5) >>> i (0.0, 0.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0) The progress meter: the higher the value of `verbose`, the more messages: >>> from time import sleep >>> from joblib import Parallel, delayed >>> r = Parallel(n_jobs=2, verbose=10)( ... delayed(sleep)(.2) for _ in range(10)) #doctest: +SKIP [Parallel(n_jobs=2)]: Done 1 tasks | elapsed: 0.6s [Parallel(n_jobs=2)]: Done 4 tasks | elapsed: 0.8s [Parallel(n_jobs=2)]: Done 10 out of 10 | elapsed: 1.4s finished Traceback example, note how the line of the error is indicated as well as the values of the parameter passed to the function that triggered the exception, even though the traceback happens in the child process: >>> from heapq import nlargest >>> from joblib import Parallel, delayed >>> Parallel(n_jobs=2)( ... delayed(nlargest)(2, n) for n in (range(4), 'abcde', 3)) ... # doctest: +SKIP ----------------------------------------------------------------------- Sub-process traceback: ----------------------------------------------------------------------- TypeError Mon Nov 12 11:37:46 2012 PID: 12934 Python 2.7.3: /usr/bin/python ........................................................................ /usr/lib/python2.7/heapq.pyc in nlargest(n=2, iterable=3, key=None) 419 if n >= size: 420 return sorted(iterable, key=key, reverse=True)[:n] 421 422 # When key is none, use simpler decoration 423 if key is None: --> 424 it = izip(iterable, count(0,-1)) # decorate 425 result = _nlargest(n, it) 426 return map(itemgetter(0), result) # undecorate 427 428 # General case, slowest method TypeError: izip argument #1 must support iteration _______________________________________________________________________ Using pre_dispatch in a producer/consumer situation, where the data is generated on the fly. Note how the producer is first called 3 times before the parallel loop is initiated, and then called to generate new data on the fly: >>> from math import sqrt >>> from joblib import Parallel, delayed >>> def producer(): ... for i in range(6): ... print('Produced %s' % i) ... yield i >>> out = Parallel(n_jobs=2, verbose=100, pre_dispatch='1.5*n_jobs')( ... delayed(sqrt)(i) for i in producer()) #doctest: +SKIP Produced 0 Produced 1 Produced 2 [Parallel(n_jobs=2)]: Done 1 jobs | elapsed: 0.0s Produced 3 [Parallel(n_jobs=2)]: Done 2 jobs | elapsed: 0.0s Produced 4 [Parallel(n_jobs=2)]: Done 3 jobs | elapsed: 0.0s Produced 5 [Parallel(n_jobs=2)]: Done 4 jobs | elapsed: 0.0s [Parallel(n_jobs=2)]: Done 6 out of 6 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=2)]: Done 6 out of 6 | elapsed: 0.0s finished ''' def __init__( self, n_jobs=default_parallel_config["n_jobs"], backend=default_parallel_config['backend'], return_as="list", verbose=default_parallel_config["verbose"], timeout=None, pre_dispatch='2 * n_jobs', batch_size='auto', temp_folder=default_parallel_config["temp_folder"], max_nbytes=default_parallel_config["max_nbytes"], mmap_mode=default_parallel_config["mmap_mode"], prefer=default_parallel_config["prefer"], require=default_parallel_config["require"], ): # Initiate parent Logger class state super().__init__() # Interpret n_jobs=None as 'unset' if n_jobs is None: n_jobs = default_parallel_config["n_jobs"] active_backend, context_config = _get_active_backend( prefer=prefer, require=require, verbose=verbose ) nesting_level = active_backend.nesting_level self.verbose = _get_config_param(verbose, context_config, "verbose") self.timeout = timeout self.pre_dispatch = pre_dispatch if return_as not in {"list", "generator"}: raise ValueError( 'Expected `return_as` parameter to be a string equal to "list"' f' or "generator", but got {return_as} instead' ) self.return_as = return_as self.return_generator = return_as != "list" # Check if we are under a parallel_config or parallel_backend # context manager and use the config from the context manager # for arguments that are not explicitly set. self._backend_args = { k: _get_config_param(param, context_config, k) for param, k in [ (max_nbytes, "max_nbytes"), (temp_folder, "temp_folder"), (mmap_mode, "mmap_mode"), (prefer, "prefer"), (require, "require"), (verbose, "verbose"), ] } if isinstance(self._backend_args["max_nbytes"], str): self._backend_args["max_nbytes"] = memstr_to_bytes( self._backend_args["max_nbytes"] ) self._backend_args["verbose"] = max( 0, self._backend_args["verbose"] - 50 ) if DEFAULT_MP_CONTEXT is not None: self._backend_args['context'] = DEFAULT_MP_CONTEXT elif hasattr(mp, "get_context"): self._backend_args['context'] = mp.get_context() if backend is default_parallel_config['backend'] or backend is None: backend = active_backend elif isinstance(backend, ParallelBackendBase): # Use provided backend as is, with the current nesting_level if it # is not set yet. if backend.nesting_level is None: backend.nesting_level = nesting_level elif hasattr(backend, 'Pool') and hasattr(backend, 'Lock'): # Make it possible to pass a custom multiprocessing context as # backend to change the start method to forkserver or spawn or # preload modules on the forkserver helper process. self._backend_args['context'] = backend backend = MultiprocessingBackend(nesting_level=nesting_level) elif backend not in BACKENDS and backend in MAYBE_AVAILABLE_BACKENDS: warnings.warn( f"joblib backend '{backend}' is not available on " f"your system, falling back to {DEFAULT_BACKEND}.", UserWarning, stacklevel=2) BACKENDS[backend] = BACKENDS[DEFAULT_BACKEND] backend = BACKENDS[DEFAULT_BACKEND](nesting_level=nesting_level) else: try: backend_factory = BACKENDS[backend] except KeyError as e: raise ValueError("Invalid backend: %s, expected one of %r" % (backend, sorted(BACKENDS.keys()))) from e backend = backend_factory(nesting_level=nesting_level) n_jobs = _get_config_param(n_jobs, context_config, "n_jobs") if n_jobs is None: # No specific context override and no specific value request: # default to the default of the backend. n_jobs = backend.default_n_jobs self.n_jobs = n_jobs if (require == 'sharedmem' and not getattr(backend, 'supports_sharedmem', False)): raise ValueError("Backend %s does not support shared memory" % backend) if (batch_size == 'auto' or isinstance(batch_size, Integral) and batch_size > 0): self.batch_size = batch_size else: raise ValueError( "batch_size must be 'auto' or a positive integer, got: %r" % batch_size) if not isinstance(backend, SequentialBackend): if self.return_generator and not backend.supports_return_generator: raise ValueError( "Backend {} does not support " "return_as={}".format(backend, return_as) ) # This lock is used to coordinate the main thread of this process # with the async callback thread of our the pool. self._lock = threading.RLock() self._jobs = collections.deque() self._pending_outputs = list() self._ready_batches = queue.Queue() self._reducer_callback = None # Internal variables self._backend = backend self._running = False self._managed_backend = False self._id = uuid4().hex self._call_ref = None def __enter__(self): self._managed_backend = True self._calling = False self._initialize_backend() return self def __exit__(self, exc_type, exc_value, traceback): self._managed_backend = False if self.return_generator and self._calling: self._abort() self._terminate_and_reset() def _initialize_backend(self): """Build a process or thread pool and return the number of workers""" try: n_jobs = self._backend.configure(n_jobs=self.n_jobs, parallel=self, **self._backend_args) if self.timeout is not None and not self._backend.supports_timeout: warnings.warn( 'The backend class {!r} does not support timeout. ' "You have set 'timeout={}' in Parallel but " "the 'timeout' parameter will not be used.".format( self._backend.__class__.__name__, self.timeout)) except FallbackToBackend as e: # Recursively initialize the backend in case of requested fallback. self._backend = e.backend n_jobs = self._initialize_backend() return n_jobs def _effective_n_jobs(self): if self._backend: return self._backend.effective_n_jobs(self.n_jobs) return 1 def _terminate_and_reset(self): if hasattr(self._backend, 'stop_call') and self._calling: self._backend.stop_call() self._calling = False if not self._managed_backend: self._backend.terminate() def _dispatch(self, batch): """Queue the batch for computing, with or without multiprocessing WARNING: this method is not thread-safe: it should be only called indirectly via dispatch_one_batch. """ # If job.get() catches an exception, it closes the queue: if self._aborting: return batch_size = len(batch) self.n_dispatched_tasks += batch_size self.n_dispatched_batches += 1 dispatch_timestamp = time.time() batch_tracker = BatchCompletionCallBack( dispatch_timestamp, batch_size, self ) self._jobs.append(batch_tracker) job = self._backend.apply_async(batch, callback=batch_tracker) batch_tracker.register_job(job) def dispatch_next(self): """Dispatch more data for parallel processing This method is meant to be called concurrently by the multiprocessing callback. We rely on the thread-safety of dispatch_one_batch to protect against concurrent consumption of the unprotected iterator. """ if not self.dispatch_one_batch(self._original_iterator): self._iterating = False self._original_iterator = None def dispatch_one_batch(self, iterator): """Prefetch the tasks for the next batch and dispatch them. The effective size of the batch is computed here. If there are no more jobs to dispatch, return False, else return True. The iterator consumption and dispatching is protected by the same lock so calling this function should be thread safe. """ if self._aborting: return False batch_size = self._get_batch_size() with self._lock: # to ensure an even distribution of the workload between workers, # we look ahead in the original iterators more than batch_size # tasks - However, we keep consuming only one batch at each # dispatch_one_batch call. The extra tasks are stored in a local # queue, _ready_batches, that is looked-up prior to re-consuming # tasks from the origal iterator. try: tasks = self._ready_batches.get(block=False) except queue.Empty: # slice the iterator n_jobs * batchsize items at a time. If the # slice returns less than that, then the current batchsize puts # too much weight on a subset of workers, while other may end # up starving. So in this case, re-scale the batch size # accordingly to distribute evenly the last items between all # workers. n_jobs = self._cached_effective_n_jobs big_batch_size = batch_size * n_jobs islice = list(itertools.islice(iterator, big_batch_size)) if len(islice) == 0: return False elif (iterator is self._original_iterator and len(islice) < big_batch_size): # We reached the end of the original iterator (unless # iterator is the ``pre_dispatch``-long initial slice of # the original iterator) -- decrease the batch size to # account for potential variance in the batches running # time. final_batch_size = max(1, len(islice) // (10 * n_jobs)) else: final_batch_size = max(1, len(islice) // n_jobs) # enqueue n_jobs batches in a local queue for i in range(0, len(islice), final_batch_size): tasks = BatchedCalls(islice[i:i + final_batch_size], self._backend.get_nested_backend(), self._reducer_callback, self._pickle_cache) self._ready_batches.put(tasks) # finally, get one task. tasks = self._ready_batches.get(block=False) if len(tasks) == 0: # No more tasks available in the iterator: tell caller to stop. return False else: self._dispatch(tasks) return True def _get_batch_size(self): """Returns the effective batch size for dispatch""" if self.batch_size == 'auto': return self._backend.compute_batch_size() else: # Fixed batch size strategy return self.batch_size def _print(self, msg): """Display the message on stout or stderr depending on verbosity""" # XXX: Not using the logger framework: need to # learn to use logger better. if not self.verbose: return if self.verbose < 50: writer = sys.stderr.write else: writer = sys.stdout.write writer(f"[{self}]: {msg}\n") def _is_completed(self): """Check if all tasks have been completed""" return self.n_completed_tasks == self.n_dispatched_tasks and not ( self._iterating or self._aborting ) def print_progress(self): """Display the process of the parallel execution only a fraction of time, controlled by self.verbose. """ if not self.verbose: return elapsed_time = time.time() - self._start_time if self._is_completed(): # Make sure that we get a last message telling us we are done self._print( f"Done {self.n_completed_tasks:3d} out of " f"{self.n_completed_tasks:3d} | elapsed: " f"{short_format_time(elapsed_time)} finished" ) return # Original job iterator becomes None once it has been fully # consumed : at this point we know the total number of jobs and we are # able to display an estimation of the remaining time based on already # completed jobs. Otherwise, we simply display the number of completed # tasks. elif self._original_iterator is not None: if _verbosity_filter(self.n_dispatched_batches, self.verbose): return self._print( f"Done {self.n_completed_tasks:3d} tasks | elapsed: " f"{short_format_time(elapsed_time)}" ) else: index = self.n_completed_tasks # We are finished dispatching total_tasks = self.n_dispatched_tasks # We always display the first loop if not index == 0: # Display depending on the number of remaining items # A message as soon as we finish dispatching, cursor is 0 cursor = (total_tasks - index + 1 - self._pre_dispatch_amount) frequency = (total_tasks // self.verbose) + 1 is_last_item = (index + 1 == total_tasks) if (is_last_item or cursor % frequency): return remaining_time = (elapsed_time / index) * \ (self.n_dispatched_tasks - index * 1.0) # only display status if remaining time is greater or equal to 0 self._print( f"Done {index:3d} out of {total_tasks:3d} | elapsed: " f"{short_format_time(elapsed_time)} remaining: " f"{short_format_time(remaining_time)}" ) def _abort(self): # Stop dispatching new jobs in the async callback thread self._aborting = True # If the backend allows it, cancel or kill remaining running # tasks without waiting for the results as we will raise # the exception we got back to the caller instead of returning # any result. backend = self._backend if (not self._aborted and hasattr(backend, 'abort_everything')): # If the backend is managed externally we need to make sure # to leave it in a working state to allow for future jobs # scheduling. ensure_ready = self._managed_backend backend.abort_everything(ensure_ready=ensure_ready) self._aborted = True def _start(self, iterator, pre_dispatch): # Only set self._iterating to True if at least a batch # was dispatched. In particular this covers the edge # case of Parallel used with an exhausted iterator. If # self._original_iterator is None, then this means either # that pre_dispatch == "all", n_jobs == 1 or that the first batch # was very quick and its callback already dispatched all the # remaining jobs. self._iterating = False if self.dispatch_one_batch(iterator): self._iterating = self._original_iterator is not None while self.dispatch_one_batch(iterator): pass if pre_dispatch == "all": # The iterable was consumed all at once by the above for loop. # No need to wait for async callbacks to trigger to # consumption. self._iterating = False def _get_outputs(self, iterator, pre_dispatch): """Iterator returning the tasks' output as soon as they are ready.""" dispatch_thread_id = threading.get_ident() detach_generator_exit = False try: self._start(iterator, pre_dispatch) # first yield returns None, for internal use only. This ensures # that we enter the try/except block and start dispatching the # tasks. yield with self._backend.retrieval_context(): yield from self._retrieve() except GeneratorExit: # The generator has been garbage collected before being fully # consumed. This aborts the remaining tasks if possible and warn # the user if necessary. self._exception = True # In some interpreters such as PyPy, GeneratorExit can be raised in # a different thread than the one used to start the dispatch of the # parallel tasks. This can lead to hang when a thread attempts to # join itself. As workaround, we detach the execution of the # aborting code to a dedicated thread. We then need to make sure # the rest of the function does not call `_terminate_and_reset` # in finally. if dispatch_thread_id != threading.get_ident(): if not IS_PYPY: warnings.warn( "A generator produced by joblib.Parallel has been " "gc'ed in an unexpected thread. This behavior should " "not cause major -issues but to make sure, please " "report this warning and your use case at " "https://github.com/joblib/joblib/issues so it can " "be investigated." ) detach_generator_exit = True _parallel = self class _GeneratorExitThread(threading.Thread): def run(self): _parallel._abort() if _parallel.return_generator: _parallel._warn_exit_early() _parallel._terminate_and_reset() _GeneratorExitThread( name="GeneratorExitThread" ).start() return # Otherwise, we are in the thread that started the dispatch: we can # safely abort the execution and warn the user. self._abort() if self.return_generator: self._warn_exit_early() raise # Note: we catch any BaseException instead of just Exception instances # to also include KeyboardInterrupt except BaseException: self._exception = True self._abort() raise finally: # Store the unconsumed tasks and terminate the workers if necessary _remaining_outputs = ([] if self._exception else self._jobs) self._jobs = collections.deque() self._running = False if not detach_generator_exit: self._terminate_and_reset() while len(_remaining_outputs) > 0: batched_results = _remaining_outputs.popleft() batched_results = batched_results.get_result(self.timeout) for result in batched_results: yield result def _wait_retrieval(self): """Return True if we need to continue retriving some tasks.""" # If the input load is still being iterated over, it means that tasks # are still on the dispatch wait list and their results will need to # be retrieved later on. if self._iterating: return True # If some of the dispatched tasks are still being processed by the # workers, wait for the compute to finish before starting retrieval if self.n_completed_tasks < self.n_dispatched_tasks: return True # For backends that does not support retrieving asynchronously the # result to the main process, all results must be carefully retrieved # in the _retrieve loop in the main thread while the backend is alive. # For other backends, the actual retrieval is done asynchronously in # the callback thread, and we can terminate the backend before the # `self._jobs` result list has been emptied. The remaining results # will be collected in the `finally` step of the generator. if not self._backend.supports_retrieve_callback: if len(self._jobs) > 0: return True return False def _retrieve(self): while self._wait_retrieval(): # If the callback thread of a worker has signaled that its task # triggered an exception, or if the retrieval loop has raised an # exception (e.g. `GeneratorExit`), exit the loop and surface the # worker traceback. if self._aborting: self._raise_error_fast() break # If the next job is not ready for retrieval yet, we just wait for # async callbacks to progress. if ((len(self._jobs) == 0) or (self._jobs[0].get_status( timeout=self.timeout) == TASK_PENDING)): time.sleep(0.01) continue # We need to be careful: the job list can be filling up as # we empty it and Python list are not thread-safe by # default hence the use of the lock with self._lock: batched_results = self._jobs.popleft() # Flatten the batched results to output one output at a time batched_results = batched_results.get_result(self.timeout) for result in batched_results: self._nb_consumed += 1 yield result def _raise_error_fast(self): """If we are aborting, raise if a job caused an error.""" # Find the first job whose status is TASK_ERROR if it exists. with self._lock: error_job = next((job for job in self._jobs if job.status == TASK_ERROR), None) # If this error job exists, immediatly raise the error by # calling get_result. This job might not exists if abort has been # called directly or if the generator is gc'ed. if error_job is not None: error_job.get_result(self.timeout) def _warn_exit_early(self): """Warn the user if the generator is gc'ed before being consumned.""" ready_outputs = self.n_completed_tasks - self._nb_consumed is_completed = self._is_completed() msg = "" if ready_outputs: msg += ( f"{ready_outputs} tasks have been successfully executed " " but not used." ) if not is_completed: msg += " Additionally, " if not is_completed: msg += ( f"{self.n_dispatched_tasks - self.n_completed_tasks} tasks " "which were still being processed by the workers have been " "cancelled." ) if msg: msg += ( " You could benefit from adjusting the input task " "iterator to limit unnecessary computation time." ) warnings.warn(msg) def _get_sequential_output(self, iterable): """Separate loop for sequential output. This simplifies the traceback in case of errors and reduces the overhead of calling sequential tasks with `joblib`. """ try: self._iterating = True self._original_iterator = iterable batch_size = self._get_batch_size() if batch_size != 1: it = iter(iterable) iterable_batched = iter( lambda: tuple(itertools.islice(it, batch_size)), () ) iterable = ( task for batch in iterable_batched for task in batch ) # first yield returns None, for internal use only. This ensures # that we enter the try/except block and setup the generator. yield None # Sequentially call the tasks and yield the results. for func, args, kwargs in iterable: self.n_dispatched_batches += 1 self.n_dispatched_tasks += 1 res = func(*args, **kwargs) self.n_completed_tasks += 1 self.print_progress() yield res self._nb_consumed += 1 except BaseException: self._exception = True self._aborting = True self._aborted = True raise finally: self.print_progress() self._running = False self._iterating = False self._original_iterator = None def _reset_run_tracking(self): """Reset the counters and flags used to track the execution.""" # Makes sur the parallel instance was not previously running in a # thread-safe way. with getattr(self, '_lock', nullcontext()): if self._running: msg = 'This Parallel instance is already running !' if self.return_generator is True: msg += ( " Before submitting new tasks, you must wait for the " "completion of all the previous tasks, or clean all " "references to the output generator." ) raise RuntimeError(msg) self._running = True # Counter to keep track of the task dispatched and completed. self.n_dispatched_batches = 0 self.n_dispatched_tasks = 0 self.n_completed_tasks = 0 # Following count is incremented by one each time the user iterates # on the output generator, it is used to prepare an informative # warning message in case the generator is deleted before all the # dispatched tasks have been consumed. self._nb_consumed = 0 # Following flags are used to synchronize the threads in case one of # the tasks error-out to ensure that all workers abort fast and that # the backend terminates properly. # Set to True as soon as a worker signals that a task errors-out self._exception = False # Set to True in case of early termination following an incident self._aborting = False # Set to True after abortion is complete self._aborted = False def __call__(self, iterable): """Main function to dispatch parallel tasks.""" self._reset_run_tracking() self._start_time = time.time() if not self._managed_backend: n_jobs = self._initialize_backend() else: n_jobs = self._effective_n_jobs() if n_jobs == 1: # If n_jobs==1, run the computation sequentially and return # immediatly to avoid overheads. output = self._get_sequential_output(iterable) next(output) return output if self.return_generator else list(output) # Let's create an ID that uniquely identifies the current call. If the # call is interrupted early and that the same instance is immediately # re-used, this id will be used to prevent workers that were # concurrently finalizing a task from the previous call to run the # callback. with self._lock: self._call_id = uuid4().hex # self._effective_n_jobs should be called in the Parallel.__call__ # thread only -- store its value in an attribute for further queries. self._cached_effective_n_jobs = n_jobs if isinstance(self._backend, LokyBackend): # For the loky backend, we add a callback executed when reducing # BatchCalls, that makes the loky executor use a temporary folder # specific to this Parallel object when pickling temporary memmaps. # This callback is necessary to ensure that several Parallel # objects using the same resuable executor don't use the same # temporary resources. def _batched_calls_reducer_callback(): # Relevant implementation detail: the following lines, called # when reducing BatchedCalls, are called in a thread-safe # situation, meaning that the context of the temporary folder # manager will not be changed in between the callback execution # and the end of the BatchedCalls pickling. The reason is that # pickling (the only place where set_current_context is used) # is done from a single thread (the queue_feeder_thread). self._backend._workers._temp_folder_manager.set_current_context( # noqa self._id ) self._reducer_callback = _batched_calls_reducer_callback # self._effective_n_jobs should be called in the Parallel.__call__ # thread only -- store its value in an attribute for further queries. self._cached_effective_n_jobs = n_jobs backend_name = self._backend.__class__.__name__ if n_jobs == 0: raise RuntimeError("%s has no active worker." % backend_name) self._print( f"Using backend {backend_name} with {n_jobs} concurrent workers." ) if hasattr(self._backend, 'start_call'): self._backend.start_call() # Following flag prevents double calls to `backend.stop_call`. self._calling = True iterator = iter(iterable) pre_dispatch = self.pre_dispatch if pre_dispatch == 'all': # prevent further dispatch via multiprocessing callback thread self._original_iterator = None self._pre_dispatch_amount = 0 else: self._original_iterator = iterator if hasattr(pre_dispatch, 'endswith'): pre_dispatch = eval_expr( pre_dispatch.replace("n_jobs", str(n_jobs)) ) self._pre_dispatch_amount = pre_dispatch = int(pre_dispatch) # The main thread will consume the first pre_dispatch items and # the remaining items will later be lazily dispatched by async # callbacks upon task completions. # TODO: this iterator should be batch_size * n_jobs iterator = itertools.islice(iterator, self._pre_dispatch_amount) # Use a caching dict for callables that are pickled with cloudpickle to # improve performances. This cache is used only in the case of # functions that are defined in the __main__ module, functions that # are defined locally (inside another function) and lambda expressions. self._pickle_cache = dict() output = self._get_outputs(iterator, pre_dispatch) self._call_ref = weakref.ref(output) # The first item from the output is blank, but it makes the interpreter # progress until it enters the Try/Except block of the generator and # reach the first `yield` statement. This starts the aynchronous # dispatch of the tasks to the workers. next(output) return output if self.return_generator else list(output) def __repr__(self): return '%s(n_jobs=%s)' % (self.__class__.__name__, self.n_jobs) joblib-1.3.2/joblib/pool.py000066400000000000000000000341131446465525000155570ustar00rootroot00000000000000"""Custom implementation of multiprocessing.Pool with custom pickler. This module provides efficient ways of working with data stored in shared memory with numpy.memmap arrays without inducing any memory copy between the parent and child processes. This module should not be imported if multiprocessing is not available as it implements subclasses of multiprocessing Pool that uses a custom alternative to SimpleQueue. """ # Author: Olivier Grisel # Copyright: 2012, Olivier Grisel # License: BSD 3 clause import copyreg import sys import warnings from time import sleep try: WindowsError except NameError: WindowsError = type(None) from pickle import Pickler from pickle import HIGHEST_PROTOCOL from io import BytesIO from ._memmapping_reducer import get_memmapping_reducers from ._memmapping_reducer import TemporaryResourcesManager from ._multiprocessing_helpers import mp, assert_spawning # We need the class definition to derive from it, not the multiprocessing.Pool # factory function from multiprocessing.pool import Pool try: import numpy as np except ImportError: np = None ############################################################################### # Enable custom pickling in Pool queues class CustomizablePickler(Pickler): """Pickler that accepts custom reducers. TODO python2_drop : can this be simplified ? HIGHEST_PROTOCOL is selected by default as this pickler is used to pickle ephemeral datastructures for interprocess communication hence no backward compatibility is required. `reducers` is expected to be a dictionary with key/values being `(type, callable)` pairs where `callable` is a function that give an instance of `type` will return a tuple `(constructor, tuple_of_objects)` to rebuild an instance out of the pickled `tuple_of_objects` as would return a `__reduce__` method. See the standard library documentation on pickling for more details. """ # We override the pure Python pickler as its the only way to be able to # customize the dispatch table without side effects in Python 2.7 # to 3.2. For Python 3.3+ leverage the new dispatch_table # feature from https://bugs.python.org/issue14166 that makes it possible # to use the C implementation of the Pickler which is faster. def __init__(self, writer, reducers=None, protocol=HIGHEST_PROTOCOL): Pickler.__init__(self, writer, protocol=protocol) if reducers is None: reducers = {} if hasattr(Pickler, 'dispatch'): # Make the dispatch registry an instance level attribute instead of # a reference to the class dictionary under Python 2 self.dispatch = Pickler.dispatch.copy() else: # Under Python 3 initialize the dispatch table with a copy of the # default registry self.dispatch_table = copyreg.dispatch_table.copy() for type, reduce_func in reducers.items(): self.register(type, reduce_func) def register(self, type, reduce_func): """Attach a reducer function to a given type in the dispatch table.""" if hasattr(Pickler, 'dispatch'): # Python 2 pickler dispatching is not explicitly customizable. # Let us use a closure to workaround this limitation. def dispatcher(self, obj): reduced = reduce_func(obj) self.save_reduce(obj=obj, *reduced) self.dispatch[type] = dispatcher else: self.dispatch_table[type] = reduce_func class CustomizablePicklingQueue(object): """Locked Pipe implementation that uses a customizable pickler. This class is an alternative to the multiprocessing implementation of SimpleQueue in order to make it possible to pass custom pickling reducers, for instance to avoid memory copy when passing memory mapped datastructures. `reducers` is expected to be a dict with key / values being `(type, callable)` pairs where `callable` is a function that, given an instance of `type`, will return a tuple `(constructor, tuple_of_objects)` to rebuild an instance out of the pickled `tuple_of_objects` as would return a `__reduce__` method. See the standard library documentation on pickling for more details. """ def __init__(self, context, reducers=None): self._reducers = reducers self._reader, self._writer = context.Pipe(duplex=False) self._rlock = context.Lock() if sys.platform == 'win32': self._wlock = None else: self._wlock = context.Lock() self._make_methods() def __getstate__(self): assert_spawning(self) return (self._reader, self._writer, self._rlock, self._wlock, self._reducers) def __setstate__(self, state): (self._reader, self._writer, self._rlock, self._wlock, self._reducers) = state self._make_methods() def empty(self): return not self._reader.poll() def _make_methods(self): self._recv = recv = self._reader.recv racquire, rrelease = self._rlock.acquire, self._rlock.release def get(): racquire() try: return recv() finally: rrelease() self.get = get if self._reducers: def send(obj): buffer = BytesIO() CustomizablePickler(buffer, self._reducers).dump(obj) self._writer.send_bytes(buffer.getvalue()) self._send = send else: self._send = send = self._writer.send if self._wlock is None: # writes to a message oriented win32 pipe are atomic self.put = send else: wlock_acquire, wlock_release = ( self._wlock.acquire, self._wlock.release) def put(obj): wlock_acquire() try: return send(obj) finally: wlock_release() self.put = put class PicklingPool(Pool): """Pool implementation with customizable pickling reducers. This is useful to control how data is shipped between processes and makes it possible to use shared memory without useless copies induces by the default pickling methods of the original objects passed as arguments to dispatch. `forward_reducers` and `backward_reducers` are expected to be dictionaries with key/values being `(type, callable)` pairs where `callable` is a function that, given an instance of `type`, will return a tuple `(constructor, tuple_of_objects)` to rebuild an instance out of the pickled `tuple_of_objects` as would return a `__reduce__` method. See the standard library documentation about pickling for more details. """ def __init__(self, processes=None, forward_reducers=None, backward_reducers=None, **kwargs): if forward_reducers is None: forward_reducers = dict() if backward_reducers is None: backward_reducers = dict() self._forward_reducers = forward_reducers self._backward_reducers = backward_reducers poolargs = dict(processes=processes) poolargs.update(kwargs) super(PicklingPool, self).__init__(**poolargs) def _setup_queues(self): context = getattr(self, '_ctx', mp) self._inqueue = CustomizablePicklingQueue(context, self._forward_reducers) self._outqueue = CustomizablePicklingQueue(context, self._backward_reducers) self._quick_put = self._inqueue._send self._quick_get = self._outqueue._recv class MemmappingPool(PicklingPool): """Process pool that shares large arrays to avoid memory copy. This drop-in replacement for `multiprocessing.pool.Pool` makes it possible to work efficiently with shared memory in a numpy context. Existing instances of numpy.memmap are preserved: the child suprocesses will have access to the same shared memory in the original mode except for the 'w+' mode that is automatically transformed as 'r+' to avoid zeroing the original data upon instantiation. Furthermore large arrays from the parent process are automatically dumped to a temporary folder on the filesystem such as child processes to access their content via memmapping (file system backed shared memory). Note: it is important to call the terminate method to collect the temporary folder used by the pool. Parameters ---------- processes: int, optional Number of worker processes running concurrently in the pool. initializer: callable, optional Callable executed on worker process creation. initargs: tuple, optional Arguments passed to the initializer callable. temp_folder: (str, callable) optional If str: Folder to be used by the pool for memmapping large arrays for sharing memory with worker processes. If None, this will try in order: - a folder pointed by the JOBLIB_TEMP_FOLDER environment variable, - /dev/shm if the folder exists and is writable: this is a RAMdisk filesystem available by default on modern Linux distributions, - the default system temporary folder that can be overridden with TMP, TMPDIR or TEMP environment variables, typically /tmp under Unix operating systems. if callable: An callable in charge of dynamically resolving a temporary folder for memmapping large arrays. max_nbytes int or None, optional, 1e6 by default Threshold on the size of arrays passed to the workers that triggers automated memory mapping in temp_folder. Use None to disable memmapping of large arrays. mmap_mode: {'r+', 'r', 'w+', 'c'} Memmapping mode for numpy arrays passed to workers. See 'max_nbytes' parameter documentation for more details. forward_reducers: dictionary, optional Reducers used to pickle objects passed from master to worker processes: see below. backward_reducers: dictionary, optional Reducers used to pickle return values from workers back to the master process. verbose: int, optional Make it possible to monitor how the communication of numpy arrays with the subprocess is handled (pickling or memmapping) prewarm: bool or str, optional, "auto" by default. If True, force a read on newly memmapped array to make sure that OS pre-cache it in memory. This can be useful to avoid concurrent disk access when the same data array is passed to different worker processes. If "auto" (by default), prewarm is set to True, unless the Linux shared memory partition /dev/shm is available and used as temp folder. `forward_reducers` and `backward_reducers` are expected to be dictionaries with key/values being `(type, callable)` pairs where `callable` is a function that give an instance of `type` will return a tuple `(constructor, tuple_of_objects)` to rebuild an instance out of the pickled `tuple_of_objects` as would return a `__reduce__` method. See the standard library documentation on pickling for more details. """ def __init__(self, processes=None, temp_folder=None, max_nbytes=1e6, mmap_mode='r', forward_reducers=None, backward_reducers=None, verbose=0, context_id=None, prewarm=False, **kwargs): if context_id is not None: warnings.warn('context_id is deprecated and ignored in joblib' ' 0.9.4 and will be removed in 0.11', DeprecationWarning) manager = TemporaryResourcesManager(temp_folder) self._temp_folder_manager = manager # The usage of a temp_folder_resolver over a simple temp_folder is # superfluous for multiprocessing pools, as they don't get reused, see # get_memmapping_executor for more details. We still use it for code # simplicity. forward_reducers, backward_reducers = \ get_memmapping_reducers( temp_folder_resolver=manager.resolve_temp_folder_name, max_nbytes=max_nbytes, mmap_mode=mmap_mode, forward_reducers=forward_reducers, backward_reducers=backward_reducers, verbose=verbose, unlink_on_gc_collect=False, prewarm=prewarm) poolargs = dict( processes=processes, forward_reducers=forward_reducers, backward_reducers=backward_reducers) poolargs.update(kwargs) super(MemmappingPool, self).__init__(**poolargs) def terminate(self): n_retries = 10 for i in range(n_retries): try: super(MemmappingPool, self).terminate() break except OSError as e: if isinstance(e, WindowsError): # Workaround occasional "[Error 5] Access is denied" issue # when trying to terminate a process under windows. sleep(0.1) if i + 1 == n_retries: warnings.warn("Failed to terminate worker processes in" " multiprocessing pool: %r" % e) # Clean up the temporary resources as the workers should now be off. self._temp_folder_manager._clean_temporary_resources() @property def _temp_folder(self): # Legacy property in tests. could be removed if we refactored the # memmapping tests. SHOULD ONLY BE USED IN TESTS! # We cache this property because it is called late in the tests - at # this point, all context have been unregistered, and # resolve_temp_folder_name raises an error. if getattr(self, '_cached_temp_folder', None) is not None: return self._cached_temp_folder else: self._cached_temp_folder = self._temp_folder_manager.resolve_temp_folder_name() # noqa return self._cached_temp_folder joblib-1.3.2/joblib/test/000077500000000000000000000000001446465525000152115ustar00rootroot00000000000000joblib-1.3.2/joblib/test/__init__.py000066400000000000000000000000001446465525000173100ustar00rootroot00000000000000joblib-1.3.2/joblib/test/_openmp_test_helper/000077500000000000000000000000001446465525000212445ustar00rootroot00000000000000joblib-1.3.2/joblib/test/_openmp_test_helper/.gitignore000066400000000000000000000000201446465525000232240ustar00rootroot00000000000000*.c *.so /build joblib-1.3.2/joblib/test/_openmp_test_helper/__init__.py000066400000000000000000000000001446465525000233430ustar00rootroot00000000000000joblib-1.3.2/joblib/test/_openmp_test_helper/parallel_sum.pyx000066400000000000000000000006061446465525000244700ustar00rootroot00000000000000import cython cimport openmp from cython.parallel import prange from libc.stdlib cimport malloc, free def parallel_sum(int N): cdef double Ysum = 0 cdef int i, num_threads cdef double* X = malloc(N*cython.sizeof(double)) for i in prange(N, nogil=True): num_threads = openmp.omp_get_num_threads() Ysum += X[i] free(X) return num_threads joblib-1.3.2/joblib/test/_openmp_test_helper/setup.py000066400000000000000000000013551446465525000227620ustar00rootroot00000000000000"""Helper module to test OpenMP support on Continuous Integration """ import os import sys from distutils.core import setup from Cython.Build import cythonize from distutils.extension import Extension if sys.platform == "darwin": os.environ["CC"] = "gcc-4.9" os.environ["CXX"] = "g++-4.9" if sys.platform != "win32": extra_compile_args = ["-ffast-math", "-fopenmp"] extra_link_args = ['-fopenmp'] else: extra_compile_args = ["/openmp"] extra_link_args = None ext_modules = [ Extension( "parallel_sum", ["parallel_sum.pyx"], extra_compile_args=extra_compile_args, extra_link_args=extra_link_args ) ] setup( name='_openmp_test_helper', ext_modules=cythonize(ext_modules), ) joblib-1.3.2/joblib/test/common.py000066400000000000000000000044401446465525000170550ustar00rootroot00000000000000""" Small utilities for testing. """ import os import gc import sys from joblib._multiprocessing_helpers import mp from joblib.testing import SkipTest, skipif try: import lz4 except ImportError: lz4 = None IS_PYPY = hasattr(sys, "pypy_version_info") # A decorator to run tests only when numpy is available try: import numpy as np def with_numpy(func): """A decorator to skip tests requiring numpy.""" return func except ImportError: def with_numpy(func): """A decorator to skip tests requiring numpy.""" def my_func(): raise SkipTest('Test requires numpy') return my_func np = None # TODO: Turn this back on after refactoring yield based tests in test_hashing # with_numpy = skipif(not np, reason='Test requires numpy.') # we use memory_profiler library for memory consumption checks try: from memory_profiler import memory_usage def with_memory_profiler(func): """A decorator to skip tests requiring memory_profiler.""" return func def memory_used(func, *args, **kwargs): """Compute memory usage when executing func.""" gc.collect() mem_use = memory_usage((func, args, kwargs), interval=.001) return max(mem_use) - min(mem_use) except ImportError: def with_memory_profiler(func): """A decorator to skip tests requiring memory_profiler.""" def dummy_func(): raise SkipTest('Test requires memory_profiler.') return dummy_func memory_usage = memory_used = None def force_gc_pypy(): # The gc in pypy can be delayed. Force it to test the behavior when it # will eventually be collected. if IS_PYPY: # Run gc.collect() twice to make sure the weakref is collected, as # mentionned in the pypy doc: # https://doc.pypy.org/en/latest/config/objspace.usemodules._weakref.html import gc gc.collect() gc.collect() with_multiprocessing = skipif( mp is None, reason='Needs multiprocessing to run.') with_dev_shm = skipif( not os.path.exists('/dev/shm'), reason='This test requires a large /dev/shm shared memory fs.') with_lz4 = skipif(lz4 is None, reason='Needs lz4 compression to run') without_lz4 = skipif( lz4 is not None, reason='Needs lz4 not being installed to run') joblib-1.3.2/joblib/test/data/000077500000000000000000000000001446465525000161225ustar00rootroot00000000000000joblib-1.3.2/joblib/test/data/__init__.py000066400000000000000000000000001446465525000202210ustar00rootroot00000000000000joblib-1.3.2/joblib/test/data/create_numpy_pickle.py000066400000000000000000000066041446465525000225240ustar00rootroot00000000000000""" This script is used to generate test data for joblib/test/test_numpy_pickle.py """ import sys import re # pytest needs to be able to import this module even when numpy is # not installed try: import numpy as np except ImportError: np = None import joblib def get_joblib_version(joblib_version=joblib.__version__): """Normalize joblib version by removing suffix. >>> get_joblib_version('0.8.4') '0.8.4' >>> get_joblib_version('0.8.4b1') '0.8.4' >>> get_joblib_version('0.9.dev0') '0.9' """ matches = [re.match(r'(\d+).*', each) for each in joblib_version.split('.')] return '.'.join([m.group(1) for m in matches if m is not None]) def write_test_pickle(to_pickle, args): kwargs = {} compress = args.compress method = args.method joblib_version = get_joblib_version() py_version = '{0[0]}{0[1]}'.format(sys.version_info) numpy_version = ''.join(np.__version__.split('.')[:2]) # The game here is to generate the right filename according to the options. body = '_compressed' if (compress and method == 'zlib') else '' if compress: if method == 'zlib': kwargs['compress'] = True extension = '.gz' else: kwargs['compress'] = (method, 3) extension = '.pkl.{}'.format(method) if args.cache_size: kwargs['cache_size'] = 0 body += '_cache_size' else: extension = '.pkl' pickle_filename = 'joblib_{}{}_pickle_py{}_np{}{}'.format( joblib_version, body, py_version, numpy_version, extension) try: joblib.dump(to_pickle, pickle_filename, **kwargs) except Exception as e: # With old python version (=< 3.3.), we can arrive there when # dumping compressed pickle with LzmaFile. print("Error: cannot generate file '{}' with arguments '{}'. " "Error was: {}".format(pickle_filename, kwargs, e)) else: print("File '{}' generated successfully.".format(pickle_filename)) if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description="Joblib pickle data " "generator.") parser.add_argument('--cache_size', action="store_true", help="Force creation of companion numpy " "files for pickled arrays.") parser.add_argument('--compress', action="store_true", help="Generate compress pickles.") parser.add_argument('--method', type=str, default='zlib', choices=['zlib', 'gzip', 'bz2', 'xz', 'lzma', 'lz4'], help="Set compression method.") # We need to be specific about dtypes in particular endianness # because the pickles can be generated on one architecture and # the tests run on another one. See # https://github.com/joblib/joblib/issues/279. to_pickle = [np.arange(5, dtype=np.dtype('77P4 4f,P:|Ba S, E N *:UéI`7Jffegd [ 1=JJC-(X*T(PRB̭K;K _[S ⒢B0cfh-d eL*dn/d *dfΆ}L@q@Tu7x:c y2'&%k2& x3f0{3 &0_@PHXDTL\BRJZFVN^AQIYEUM]CSK[GWO? 0(8$4,<"2*:&6.>耔Դ̬ܼ¢ҲʪںƦֶή޾ 'M2u3g͞3w -^t+W^v 7m޲u;w޳w>r'O>s /]r7oݾs=~/_~?}?B%pP-T%`þPM,)ʬA_ .(USy?0SKr,9RAP=U`kjoblib-1.3.2/joblib/test/data/joblib_0.10.0_compressed_pickle_py27_np17.gz000066400000000000000000000013651446465525000260270ustar00rootroot00000000000000x^k`-dHOL+-/LIq+Ë R 5 j 5BYSJ* R YZ77P4 4f,P:|Ba S, E N *:UéI`7Jffegd [ 1=JJC-(X*T(PRB̭K;K _[S ⒢B0cfh-d eL*dn/d *dfΆ}L@q@Tu7x:c y2'&%k2& x3f0{3 &0_@PHXDTL\BRJZFVN^AQIYEUM]CSK[GWO? 0(8$4,<"2*:&6.>耔Դ̬ܼ¢ҲʪںƦֶή޾ 'M2u3g͞3w -^t+W^v 7m޲u;w޳w>r'O>s /]r7oݾs=~/_~?}?B%pP-T%`þPM,)ʬA_ .(USy?0SKr,9RAP=UwTjoblib-1.3.2/joblib/test/data/joblib_0.10.0_compressed_pickle_py33_np18.gz000066400000000000000000000014301446465525000260160ustar00rootroot00000000000000x^eSexPn]&a+Vn0VtCpÒ({MPB'nx)#=3:mt S<{)RbPQGV,+rY +dȍ+Ci$0/Yu#NF qi>"md҂(ǴP8iAZ YI* v +.Gུ_"Tvڃ1Ngh)؝ALRe`ATL\L²WJOpCR^*IZ2DPR3*F ‚CB9sEΓ7_ .UX%J)[TBJTV=FZԍW?AF4m7oan٪Mv;;u i1{8#x{دAI >|Qnj7~IL6}Y̝7E,X|U׬]~Mlݶ}]ݗC=vSϜ=wK\v[ܽwG<}W߼}O|_% OQG'!jE g W|ԨyJdGG3!23"$c _joblib-1.3.2/joblib/test/data/joblib_0.10.0_compressed_pickle_py34_np19.gz000066400000000000000000000014321446465525000260220ustar00rootroot00000000000000x^eSexPn]&a+Vn0e I Zo@Moju D :I%{g]}8()y)iÆ9jO8iӦϘ9k/Xh̥˖Xjoظi۶عkY8xǎ8y_x׮߸y?xϞx߾ QRA1ytfLR}AF(U+Jdk#Ege YQP1:_joblib-1.3.2/joblib/test/data/joblib_0.10.0_compressed_pickle_py35_np19.gz000066400000000000000000000014261446465525000260260ustar00rootroot00000000000000x^uSe|P]I;؆+C76tEd$ ʥ)00dw)2>K/w/E2=A0V`wۜfPQgY<9ŝNE@^z (*T kXPcr~; !҈CSGh@P*OQc4 ? C#a6m( ].y\`@ !` foYy?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~q"h)q#}q$(hhh KKq%h hhcnumpy.matrixlib.defmatrix matrix q&hubXC'est l'été !q'e.joblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py27_np17.pkl.bz2000066400000000000000000000017451446465525000244470ustar00rootroot00000000000000BZh91AY&SYrG(\1EU4&&4LL1=``L4`40IdzLɓzG0j4i '2H FLmFh !4ɠdi)4B1SOI4id 4 LOSF@4 @C#FhL& hE4hm hh4 4ѠF@ɡhh = aL"ğ Bj ԑ@f ܰg(-Kd#V@E P4 bvaF*C2!ql8PfᩝÙ DAB (gL:iӎuH]w7&8NOw%2WgdNl "ne{Pr'{@_U߯Qr%YD!!k)0 [aCa1ˁ>&[D@Ph CRW V(Z O7opqr%tu k |-Uy p`CEő-Z䐥W:(uJ3V[38H&4"!d&4]dLċȗ̴IV8dH2jx-+7E y<4;;$(% ri3]z&H -a4z[Rg%)VkB NBPBh9*`PE V${n6qE#sE={p93:ͦ2F2Uyzny*'&MsJ ,'!@,\26:>BFJNRV[/13579;=?ACEGIK GMOUWY[]_ac~lVv,Q"/o0p1q2ryy:Zz]m}sw7$p@.p!Pjoblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py27_np17.pkl.gzip000066400000000000000000000014361446465525000247200ustar00rootroot00000000000000zVjoblib_0.10.0_pickle_py27_np17.pklk`-dHOL+-/LIq+Ë R 5 j 5BYSJ* R YZ77P4 4f,P:|Ba S, E N *:UéI`7Jffegd [  JJC-(X*T(PRB̭K;K _[S ⒢B0cfh-d eL*dn/d *dfΆ}`q@Tu7x:c y2'&%k2& x3f0{3 &酀_@PHXDTL\BRJZFVN^AQIYEUM]CSK[GWO? 0(8$4,<"2*:&6.>耔Դ̬ܼ¢ҲʪںƦֶή޾ 'M2u3g͞3w -^t+W^v 7m޲u;w޳w>r'O>s /]r7oݾs=~/_~?}?B%pP-T%`þPM,)ʬA_ .(U/G g+KTP,TO`kjoblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py27_np17.pkl.lzma000066400000000000000000000012241446465525000247050ustar00rootroot00000000000000]@I׀qm&˻1)'vQɨ˗)߀y=@-kMeqgͅܧ?Iu?^hѭ(}: );(lHnnACN>5Xgsw)DEZ4̒̀娅aq"u|#T9 _DIOU`B}s;_3y&R(~ BcXiNKG! -Ɂu"S'FϤ yH?9m![E3]U{W E#5(R:ѽT6E- Tc~Z/r2/\KJmA-%;{C?E ǖ0Racf[VpR5qtCQG^5Xgsw)DEZ4̒̀娅aq"u|#T9 _DIOU`B}s;_3y&R(~ BcXiNKG! -Ɂu"S'FϤ yH?9m![E3]U{W E#5(R:ѽT6E- Tc~Z/r2/\KJmA-%;{C?E ǖ0Racf[VpR5qtCQG^׳0gYZjoblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py33_np18.pkl000066400000000000000000000020541446465525000237430ustar00rootroot00000000000000]q(cjoblib.numpy_pickle NumpyArrayWrapper q)q}q(XshapeqKqX allow_mmapqXsubclassqcnumpy ndarray qXdtypeq cnumpy dtype q Xi8q KKq Rq (KX?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~q!h)q"}q#(hKKq$hhcnumpy.matrixlib.defmatrix matrix q%h h hhubXC'est l'été !q&e.joblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py33_np18.pkl.bz2000066400000000000000000000017501446465525000244410ustar00rootroot00000000000000BZh91AY&SY|u*B12i0 LM04U= &dF&H& f` 2b``&LLidɦ")h䧚Q3H hj=GG  5 SOjiCO4h44Aeީae٥ppJ(S,b lB , .TB*(2ĴB &P c{6&BFJNRVZ^bfjnrvz~ j+k,l-m.n/o֡0 0p1q2r3s4t5uw{g_xC]BBn\joblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py33_np18.pkl.gzip000066400000000000000000000014771446465525000247230ustar00rootroot00000000000000zVjoblib_0.10.0_pickle_py33_np18.pkluexPvݺL oV0tK@"$i TN 0݇pwwwwZ{MQ%#{:Jn3촒 5,'w`qbQPJT:AkX1)v}Pc.wRτ88m6 LFX&Hwy݂ Pq o O=MJ~A)2 Zٵ `04ɇB 7ȡU)JVYiXaȣ}h5-5?:h_hP)^%ȍDI4$(Ҡ m$ha#w͍N G(;ӓ g.RX7ɡ} I%i&t]h >28gǁ:|#0`͝@z[c=JP~/ ?8c 7A'&ST}4h@`P6$4,OxD| ,THѨbKՕ*]l9} q+URZ|5kծX^ 5nҴּe6m۵`ةsn սmًN`]w :xд̬aG5zq'L4yig̜5{y,\xIeW\zu7lܴymwܵ{ޜ}Uغ:YP-"l/蔥NcKO}xZspK8YyY-Ktr|ǚQ͂~1>`ᘻ˳19{VZXiv`mKF aA[.'cm3`B٥g$T|}nkϏ-Oneed iD1׽zJ[B}`7 ĠsOgk@Q**r};ufמ3]WWgƬ:Y (zp,'5Jlj 3/5S/֒Q[)(9c EiqDƊՉ^xi=٠ qZE}a!zud!a~3*q_lAljDAL ۥٵ%^*v֐bt]e0J]%& nTPޟ 7 z(q$ =/`֩䃢S~} ,)p7g¹CX\OTz%T%$BF7ъg44/%Ҕ0^D:,䇝BVթU]7y-[F ܹ 6%0(ɫE,jJ ¬Z|nF=vvbtyw噉)<[w@9CTnh4ZuzhgYZjoblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py34_np19.pkl000066400000000000000000000020541446465525000237450ustar00rootroot00000000000000]q(cjoblib.numpy_pickle NumpyArrayWrapper q)q}q(Xdtypeqcnumpy dtype qXi8qKKqRq(KX?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~q!h)q"}q#(hhh cnumpy.matrixlib.defmatrix matrix q$h hhhKKq%ubXC'est l'été !q&e.joblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py34_np19.pkl.bz2000066400000000000000000000017751446465525000244520ustar00rootroot00000000000000BZh91AY&SYz!|QyL4&L I`# 10  4L10A!=CІ#z"dƚ'j4='iO&SɄ=FG FѠLM0&#!M#&@4hC*z)43S&M4=@A2@@4ԂH4M3SMhh4d @4 (,F`6ia-\;KȈb/?+EPB K , &$'#eTˤ l&it$ xF1B.-Lq5v:"vCS\,z"zD}/ɡD0=QhIeѬ̛.eTv "-:<>89:gv]ADd@Љ1r|}~QTHFxO$\\?RP J DvD ` 6 (6Mdqnd bFIק#xL#3((ja#W}$TXhȃT1OBfӤbL:5y0֜83{`!VdeŸyp*lq kQ{5-J2Р!aȀ& OZ4;" MA2@D0l\'V&L'%yhc~~ UR"$+YfD2]42*nJB(cTX⑔`େLyKInrmPcO n0IH@B$!!"b#c$d%eLMNOPQ*5=EMU]emu}zG #'+/37;?CGHN'?0?G󧫯L;)„ joblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py34_np19.pkl.gzip000066400000000000000000000014771446465525000247250ustar00rootroot00000000000000zVjoblib_0.10.0_pickle_py34_np19.pkluu|@S]I7pٰ+C7(P,(IH!_ 0݇pwwwwO ۇ#rwwUNFJlTOFMF]#sEs((J& *C!)pAMQ GAkNJsBᨠ:{[v4q!\4cd@+ tk-q A)R! z׮!& !\*eUɪm+r5aKcDGaL#ECeT*/cJ$:YCbEh^/.@UɐE4P(k"tR:3#h6) GSN<oZ I%eVrWAAƵosTM h :30/wTY%Afm]zZ/0!4Ɖa**FG!9rʝ'o *Yh% %K.SX|*WZ-zj׉W~7i5knjѲu:vU`ugl=zv'o >t#G3v 'M2u3g͞3w -Xt+W^v 7m޲u;wޓw>r'O>s /]r7oݾs=~/_~?})^"^(*9[#K T(N @q>pFn>33""m e,joblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py34_np19.pkl.lzma000066400000000000000000000012711446465525000247070ustar00rootroot00000000000000],@I׀qm&˻1)'vQɨ˗)߀y=@-kMe5aKKN&5Pu\0FU֚9F,񷊶ѬwRb&$Y\L~~}HƸjx.߆Za3h7 a9.c(=Lnku,LCU%p=%(6r}w;ﱢ) Q}OTvy%Y,V*K= ێ*բI]jL5fӎ]%6G+3=ϭ3Yoez=zj(S\}ў䋗\2Q)6C*b\ %4mO2AtنԦe${P"x4hDD!g\ wI&$7,"櫻S:g_珸|d3әEZOZG# > Γdd@ L{=A!0yXpLQ˘u!t8qLz^tq2Us')PZTGyŲ/ZRr<=':j?/F NOT?^󐓆Pz@n= A}]2hgdm Aψ:8X5 ͏ub\V83+G?zpjoblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py34_np19.pkl.xz000066400000000000000000000013601446465525000244040ustar00rootroot000000000000007zXZִF!t/+]@I׀qm&˻1)'vQɨ˗)߀y=@-kMe4clSb9x 4*_ae{"d"GoOӴ$^ 6b/D ֫)8*d6FRfn2!={ZXR Dv!YȺz˒B-c7x"YйѮ~}(4̞>vi@J Ӷ!Z@Wo^#5IY?~2 uAjTs/Zzɒ'Hy5hfXT2nyhTM)d! d뵜uPlWLCV&/ML@α]t"-&14qRPpB^W?=.hʐ-;(%1Ձ jMhO!A#<{p^yhAY1oгx/ŸmB s c-}/wX q܎zQD(+Ost/lG5 aB98VD$=3P-m1}&2c`C5s# K4Iv<;a"(i |i$7}LϨ MOH XbϠ k-7L&;#"iEE' {Z䳠^# Z^\ZQ?밤C o.x :Agr1dgYZjoblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py35_np19.pkl000066400000000000000000000020541446465525000237460ustar00rootroot00000000000000]q(cjoblib.numpy_pickle NumpyArrayWrapper q)q}q(X allow_mmapqXshapeqKqXorderqXCqXsubclassq cnumpy ndarray q Xdtypeq cnumpy dtype q Xi8q KKqRq(KX?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~q!h)q"}q#(hhKKq$hhh cnumpy.matrixlib.defmatrix matrix q%h hubXC'est l'été !q&e.joblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py35_np19.pkl.bz2000066400000000000000000000017551446465525000244510ustar00rootroot00000000000000BZh91AY&SYG|Q4L&0&d=b `&M0!M@mT1=G4' )zO&`@@dhb2dA&M240FL 4р"5cRh3SL@d@ 44~z6i4Mz f![#NB8GȈ.?*@( k 6- l5m[mLL%9DPR $p*l2PIT9=hzQe?IZIbKT0L$2&LH#33Cf0lfYч3&MfXsÛ0X=Hq؄@N.[m6QtuM燒G(O!T%DUդ%{|YH| 6"d&LB@:ċ@P‡iQ~)3FT0%P7 =V:rsEnW l!*|͠7`\%?`Cx@I-'g9%@!)p!PpX Q A7C. BpT Xb.!T (UFBY̿?2A!;l/E zC%^(C" ʤ+iq<ZLhLQy2RN6=,oehpQ@Φ C@MY((($hqY9*.|z30Ȏ 4CPN0Bf&A. )yB(bjCi'Shu>z?? 0(8$4W}yHI8hpZzƐ̡Æ9jO8iӦϘ9k/Xhq֒˖Xjoظi۶عk8xǎ8y_x׮߸y?xϞx߾(yPTl Jz, ;PLbP|n>ⳳ"m Rޫ_,joblib-1.3.2/joblib/test/data/joblib_0.10.0_pickle_py35_np19.pkl.lzma000066400000000000000000000012751446465525000247140ustar00rootroot00000000000000],@I׀qm&˻1)'vQɨ˗)߀y=@-kMe4clSb9x 4*_ae{"d"GoOӴ$^ 6b/D ֫)! yTkEqtF 0WȞɁXpI秓HYimI$f><j~x<>͞py~s `{ 4ALa5bY}x^z& J'7m(D_񒘟L^[- CRS} *w*c)y,}v\p]_EK/7{s9"e$*QOqHM½3\&RE~ f#yYtT]u{8N#V^~Y@1$UVDDyN}GE8ߍ] RKl'Rv?ճ573ԿQ3cMsptspv45iV@Q|"D."0g:{"`  j˩4 [9O:2yy-70V-⁢AթP,hh(.0Aɿ'C3TJ~E VY3Kݴ"ZJ\ *L;, 8@~n 4)XU@>&M GsE!TF"\( IѲ: *7#@eT˓7@`P! .RX%K(S6RU|JTVFZѵԭWALløF4mּ޲uv;tL :u5;`azdzs6 Kp/dA) >?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~q!h)q"}q#(hcnumpy.matrixlib.defmatrix matrix q$hKKq%hh h hhubXC'est l'été !q&e.joblib-1.3.2/joblib/test/data/joblib_0.11.0_pickle_py36_np111.pkl.bz2000066400000000000000000000017371446465525000245240ustar00rootroot00000000000000BZh31AY&SYf|sFqѠ000" 4ژLHhɚz =#&10OH @! LM04i05)zSOJ2i@ =@4@m@zhhh2h 4h4C 44@Ѧ=FESQ!|+ *l舔.?*FThB͹ Bp^ 4rʖP (# 2`$3+y`-% EdV4, 6$2-p&$Le͆  Bb&C gØ93J'1*>f^QQOPox& .Aeimquy}B̬L`"4h-M|cMP`\" #Z ,%R:.>+ջEkAhSPYM mHaA >? ؉QjC[O"M+0@D1897P?056.O@cxI̓P.\MDDd! @$g[fdKab'k:XR2MPϸhtt lwrt;w ':X NV@6;ZY!#%')+-X[]_acegikmoqsuwy{}Au5ug>[&/dd? i"(H3T@joblib-1.3.2/joblib/test/data/joblib_0.11.0_pickle_py36_np111.pkl.gzip000066400000000000000000000014401446465525000247670ustar00rootroot00000000000000mSu\AcUB8[1AWO=5[ӽx{{*) vw7vwcwwww 7f}ߛM:z⬔9N+ݛc0hdRO: 2j0<:BrS4G\B1Zq% T#h~R"$C7 B[hȋ+pe:3WKAhl4 )C[$9m6 AH&M)5^ϊe@=aD8ձ~,LzDC̤ !& S쿠r71^X>0V-⁢AթP,hh(.0Aɿ'C3TJ~E VY3Kݴ"ZJ\ *L;, 8@~n 4)XU@>&M GsE!TF"\( IѲ: *7#@eT˓7@`P! .RX%K(S6RU|JTVFZѵԭWALløF4mּ޲uv;tL :u5;`azdzs6 Kp/dA) >͚iCtUёT؟8=:!2V0gQ;Ywǖ+Zv“.Fg<5~6+Us9\/u>It='G8@^u{L\?Q 9IQw¾sbV44edB-_U>'UVÉG9C E boUEعwO90`{:`1 < |+A̬a/ղ3"jtmTi7 }}id#-MQAU[;y}(#R2zi%>kK1I1;. ~07lŽV˩W̻2-!l>aCD[㴲YH& ?cBy^$ qa)B70҅RS`l gcC B"rXa-k!c54EcM-[+͚iCtUёT؟8=:!2V0gQ;Ywǖ+Zv“.Fg<5~6+Us9\/u>It='G8@^u{L\?Q 9IQw¾sbV44edB-_U>'UVÉG9C E boUEعwO90`{:`1 < |+A̬a/ղ3"jtmTi7 }}id#-MQAU[;y}(#R2zi%>kK1I1;. ~07lŽV˩W̻2-!l>aCD[㴲YH& ?cBy^$ qa)B70҅RS`l gcC B"rXa-k!c54EcM-[+?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~q!=c"}q#(h|p.matrix2def  @q$h3q%a/h  X'est l'été !q&e.joblib-1.3.2/joblib/test/data/joblib_0.8.4_compressed_pickle_py27_np17.gz000066400000000000000000000012231446465525000257530ustar00rootroot00000000000000ZF0x31f x^k`-dH+-K/J-)L,*J/JM+.)*M.*dK2y322&ix3zCTrr2eZrz3x3rrkx5y0PRțP J3Ci(]WRȟv@ ABP7gM(X+T(VZ dk!9CdIZi2A k[ @[CU*T[[Sj}IrRgl Pc(sbRrjmF(cb7c7SijIFR(YX98yxED%$edUT54ut ML-,ml]\=<}|CB#"cbgdfeWTVU7465wtvuO8iӦϘ9k/Xh˖Xjoظi۶عk8xǎ8y_x׮߸y?xϞx߾jf@^nbIQfENf*%mPb'mzܝU~IAR?PY=D!Gʒ+ Sjoblib-1.3.2/joblib/test/data/joblib_0.9.2_compressed_pickle_py27_np16.gz000066400000000000000000000012221446465525000257500ustar00rootroot00000000000000ZF0x31f x^k`-dH+-K/J-)L,*J/JM+.)*M.*dK2y322&ix3zCTrr2eZrz3x3rrkx5y0PRțP J3Ci(]WRȟv@ ABP7gM(X+T(VZ dk!9CdIZi2A k[ @[CU*T[[Sj}IrRgl Pc(sbRrjmF(cb7c7SijIFR(YX98yxED%$edUT54ut ML-,ml]\=<}|CB#"cbgdfeWTVU7465wtvuO8iӦϘ9k/Xh˖Xjoظi۶عk8xǎ8y_x׮߸y?xϞx߾jf@^nbIQfENf*%mPb'mzܝU~IAR?PY=D!Gʒ+ Sjoblib-1.3.2/joblib/test/data/joblib_0.9.2_compressed_pickle_py27_np17.gz000066400000000000000000000012221446465525000257510ustar00rootroot00000000000000ZF0x31f x^k`-dH+-K/J-)L,*J/JM+.)*M.*dK2y322&ix3zCTrr2eZrz3x3rrkx5y0PRțP J3Ci(]WRȟv@ ABP7gM(X+T(VZ dk!9CdIZi2A k[ @[CU*T[[Sj}IrRgl Pc(sbRrjmF(cb7c7SijIFR(YX98yxED%$edUT54ut ML-,ml]\=<}|CB#"cbgdfeWTVU7465wtvuO8iӦϘ9k/Xh˖Xjoظi۶عk8xǎ8y_x׮߸y?xϞx߾jf@^nbIQfENf*%mPb'mzܝU~IAR?PY=D!Gʒ+ Sjoblib-1.3.2/joblib/test/data/joblib_0.9.2_compressed_pickle_py34_np19.gz000066400000000000000000000012411446465525000257520ustar00rootroot00000000000000ZF0x337 x^k`-dH+-K/J-)L,*J/JM+.)*M.*dK2y32;3&ix3zCTrrD0100dZrz3x3rrkx3G0Em yf()MtʁHf( J 23D0P i"E ŀg`Z+l-P[ 4p(()[+V:Pd-sk,ZB9A 0?*پP)3P=ĤBBU p%y3Yތ%INadbfaecWPTRVQUS70426153wptrvqus  O:$%5-=#3+;'7/IL6}Y̝7E,]|U׬]~Mlݶ}]ݷC=vSϜ=wK\v[ܽwG<}W߼}O|_P3(rK2+r2pT(ihH'PGXj+t0FB+/)4H;(^YrxbaH5joblib-1.3.2/joblib/test/data/joblib_0.9.2_compressed_pickle_py35_np19.gz000066400000000000000000000012411446465525000257530ustar00rootroot00000000000000ZF0x337 x^k`-dH+-K/J-)L,*J/JM+.)*M.*dK2y32;3&ix3zCTrrD0100dZrz3x3rrkx3G0Em yf()MtʁHf( J 23D0P i"E ŀg`Z+l-P[ 4p(()[+V:Pd-sk,ZB9A 0?*پP)3P=ĤBBU p%y3Yތ%INadbfaecWPTRVQUS70426153wptrvqus  O:$%5-=#3+;'7/IL6}Y̝7E,]|U׬]~Mlݶ}]ݷC=vSϜ=wK\v[ܽwG<}W߼}O|_P3(rK2+r2pT(ihH'PGXj+t0FB+/)4H;(^YrxbaH5joblib-1.3.2/joblib/test/data/joblib_0.9.2_pickle_py27_np16.pkl000066400000000000000000000012361446465525000236770ustar00rootroot00000000000000]q(cjoblib.numpy_pickle NDArrayWrapper q)q}q(U allow_mmapqUsubclassqcnumpy ndarray qUfilenameqU(joblib_0.9.2_pickle_py27_np16.pkl_01.npyqubh)q }q (hhhhU(joblib_0.9.2_pickle_py27_np16.pkl_02.npyq ubh)q }q (hhhhU(joblib_0.9.2_pickle_py27_np16.pkl_03.npyqubT  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~qh)q}q(hhcnumpy.matrixlib.defmatrix matrix qhU(joblib_0.9.2_pickle_py27_np16.pkl_04.npyqubXC'est l'été !qe.joblib-1.3.2/joblib/test/data/joblib_0.9.2_pickle_py27_np16.pkl_01.npy000066400000000000000000000001701446465525000250000ustar00rootroot00000000000000NUMPYF{'descr': '?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~qh)q}q(hhcnumpy.matrixlib.defmatrix matrix qhU(joblib_0.9.2_pickle_py27_np17.pkl_04.npyqubXC'est l'été !qe.joblib-1.3.2/joblib/test/data/joblib_0.9.2_pickle_py27_np17.pkl_01.npy000066400000000000000000000001701446465525000250010ustar00rootroot00000000000000NUMPYF{'descr': '?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~qh)q}q(hhcnumpy.matrixlib.defmatrix matrix qhX(joblib_0.9.2_pickle_py33_np18.pkl_04.npyqubXC'est l'été !qe.joblib-1.3.2/joblib/test/data/joblib_0.9.2_pickle_py33_np18.pkl_01.npy000066400000000000000000000001701446465525000247770ustar00rootroot00000000000000NUMPYF{'descr': '?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~qh)q}q(hX(joblib_0.9.2_pickle_py34_np19.pkl_04.npyqhhcnumpy.matrixlib.defmatrix matrix qubXC'est l'été !qe.joblib-1.3.2/joblib/test/data/joblib_0.9.2_pickle_py34_np19.pkl_01.npy000066400000000000000000000001701446465525000250010ustar00rootroot00000000000000NUMPYF{'descr': '?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~qh)q}q(hcnumpy.matrixlib.defmatrix matrix qhhX(joblib_0.9.2_pickle_py35_np19.pkl_04.npyqubXC'est l'été !qe.joblib-1.3.2/joblib/test/data/joblib_0.9.2_pickle_py35_np19.pkl_01.npy000066400000000000000000000001701446465525000250020ustar00rootroot00000000000000NUMPYF{'descr': 'oB~gƤBB̜ԼB x=K=2܂Ԕ̪T *M -ҫ * K2@~-ȀP,4Bq$ %;`d+AA](W TҜLH&&pCrbP([ ^(TxE5P ?5c_R[  ĤBBm p|&x3Yތ%zIN F&fV6vN.n^>~A!aQ1q I)iY9yE%eU5u M-m]=}C#cS3s K+k[;{G'gW7wO/o_?аȨظCRR32sr KJ+*kj[Z;:{z'L4yig̜5{y,\xeW\zu7lܴymwܵ{} 0 @with_numpy @with_multiprocessing def test_parallel_config_params_explicit_set(tmpdir): with parallel_config(n_jobs=3, max_nbytes=1, temp_folder=tmpdir): with Parallel(n_jobs=2, prefer="processes", max_nbytes='1M') as p: assert isinstance(p._backend, LokyBackend) assert p.n_jobs == 2 # Checks that memmapping is disabled with raises(TypeError, match="Expected np.memmap instance"): p(delayed(check_memmap)(a) for a in [np.random.random(10)] * 2) @parametrize("param", ["prefer", "require"]) def test_parallel_config_bad_params(param): # Check that an error is raised when setting a wrong backend # hint or constraint with raises(ValueError, match=f"{param}=wrong is not a valid"): with parallel_config(**{param: "wrong"}): Parallel() def test_parallel_config_constructor_params(): # Check that an error is raised when backend is None # but backend constructor params are given with raises(ValueError, match="only supported when backend is not None"): with parallel_config(inner_max_num_threads=1): pass with raises(ValueError, match="only supported when backend is not None"): with parallel_config(backend_param=1): pass def test_parallel_config_nested(): # Check that nested configuration retrieves the info from the # parent config and do not reset them. with parallel_config(n_jobs=2): p = Parallel() assert isinstance(p._backend, BACKENDS[DEFAULT_BACKEND]) assert p.n_jobs == 2 with parallel_config(backend='threading'): with parallel_config(n_jobs=2): p = Parallel() assert isinstance(p._backend, ThreadingBackend) assert p.n_jobs == 2 with parallel_config(verbose=100): with parallel_config(n_jobs=2): p = Parallel() assert p.verbose == 100 assert p.n_jobs == 2 @with_numpy @with_multiprocessing @parametrize('backend', ['multiprocessing', 'threading', MultiprocessingBackend(), ThreadingBackend()]) @parametrize("context", [parallel_config, parallel_backend]) def test_threadpool_limitation_in_child_context_error(context, backend): with raises(AssertionError, match=r"does not acc.*inner_max_num_threads"): context(backend, inner_max_num_threads=1) @parametrize("context", [parallel_config, parallel_backend]) def test_parallel_n_jobs_none(context): # Check that n_jobs=None is interpreted as "unset" in Parallel # non regression test for #1473 with context(backend="threading", n_jobs=2): with Parallel(n_jobs=None) as p: assert p.n_jobs == 2 with context(backend="threading"): default_n_jobs = Parallel().n_jobs with Parallel(n_jobs=None) as p: assert p.n_jobs == default_n_jobs @parametrize("context", [parallel_config, parallel_backend]) def test_parallel_config_n_jobs_none(context): # Check that n_jobs=None is interpreted as "explicitly set" in # parallel_(config/backend) # non regression test for #1473 with context(backend="threading", n_jobs=2): with context(backend="threading", n_jobs=None): # n_jobs=None resets n_jobs to backend's default with Parallel() as p: assert p.n_jobs == 1 joblib-1.3.2/joblib/test/test_dask.py000066400000000000000000000433131446465525000175500ustar00rootroot00000000000000from __future__ import print_function, division, absolute_import import os import warnings import pytest from random import random from uuid import uuid4 from time import sleep from .. import Parallel, delayed, parallel_config from ..parallel import ThreadingBackend, AutoBatchingMixin from .._dask import DaskDistributedBackend distributed = pytest.importorskip('distributed') dask = pytest.importorskip('dask') # These imports need to be after the pytest.importorskip hence the noqa: E402 from distributed import Client, LocalCluster, get_client # noqa: E402 from distributed.metrics import time # noqa: E402 from distributed.utils_test import cluster, inc # noqa: E402 def noop(*args, **kwargs): pass def slow_raise_value_error(condition, duration=0.05): sleep(duration) if condition: raise ValueError("condition evaluated to True") def count_events(event_name, client): worker_events = client.run(lambda dask_worker: dask_worker.log) event_counts = {} for w, events in worker_events.items(): event_counts[w] = len([event for event in list(events) if event[1] == event_name]) return event_counts def test_simple(loop): with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as client: # noqa: F841 with parallel_config(backend='dask'): seq = Parallel()(delayed(inc)(i) for i in range(10)) assert seq == [inc(i) for i in range(10)] with pytest.raises(ValueError): Parallel()(delayed(slow_raise_value_error)(i == 3) for i in range(10)) seq = Parallel()(delayed(inc)(i) for i in range(10)) assert seq == [inc(i) for i in range(10)] def test_dask_backend_uses_autobatching(loop): assert (DaskDistributedBackend.compute_batch_size is AutoBatchingMixin.compute_batch_size) with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as client: # noqa: F841 with parallel_config(backend='dask'): with Parallel() as parallel: # The backend should be initialized with a default # batch size of 1: backend = parallel._backend assert isinstance(backend, DaskDistributedBackend) assert backend.parallel is parallel assert backend._effective_batch_size == 1 # Launch many short tasks that should trigger # auto-batching: parallel( delayed(lambda: None)() for _ in range(int(1e4)) ) assert backend._effective_batch_size > 10 def random2(): return random() def test_dont_assume_function_purity(loop): with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as client: # noqa: F841 with parallel_config(backend='dask'): x, y = Parallel()(delayed(random2)() for i in range(2)) assert x != y @pytest.mark.parametrize("mixed", [True, False]) def test_dask_funcname(loop, mixed): from joblib._dask import Batch if not mixed: tasks = [delayed(inc)(i) for i in range(4)] batch_repr = 'batch_of_inc_4_calls' else: tasks = [ delayed(abs)(i) if i % 2 else delayed(inc)(i) for i in range(4) ] batch_repr = 'mixed_batch_of_inc_4_calls' assert repr(Batch(tasks)) == batch_repr with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as client: with parallel_config(backend='dask'): _ = Parallel(batch_size=2, pre_dispatch='all')(tasks) def f(dask_scheduler): return list(dask_scheduler.transition_log) batch_repr = batch_repr.replace('4', '2') log = client.run_on_scheduler(f) assert all('batch_of_inc' in tup[0] for tup in log) def test_no_undesired_distributed_cache_hit(loop): # Dask has a pickle cache for callables that are called many times. Because # the dask backends used to wrap both the functions and the arguments # under instances of the Batch callable class this caching mechanism could # lead to bugs as described in: https://github.com/joblib/joblib/pull/1055 # The joblib-dask backend has been refactored to avoid bundling the # arguments as an attribute of the Batch instance to avoid this problem. # This test serves as non-regression problem. # Use a large number of input arguments to give the AutoBatchingMixin # enough tasks to kick-in. lists = [[] for _ in range(100)] np = pytest.importorskip('numpy') X = np.arange(int(1e6)) def isolated_operation(list_, data=None): if data is not None: np.testing.assert_array_equal(data, X) list_.append(uuid4().hex) return list_ cluster = LocalCluster(n_workers=1, threads_per_worker=2) client = Client(cluster) try: with parallel_config(backend='dask'): # dispatches joblib.parallel.BatchedCalls res = Parallel()( delayed(isolated_operation)(list_) for list_ in lists ) # The original arguments should not have been mutated as the mutation # happens in the dask worker process. assert lists == [[] for _ in range(100)] # Here we did not pass any large numpy array as argument to # isolated_operation so no scattering event should happen under the # hood. counts = count_events('receive-from-scatter', client) assert sum(counts.values()) == 0 assert all([len(r) == 1 for r in res]) with parallel_config(backend='dask'): # Append a large array which will be scattered by dask, and # dispatch joblib._dask.Batch res = Parallel()( delayed(isolated_operation)(list_, data=X) for list_ in lists ) # This time, auto-scattering should have kicked it. counts = count_events('receive-from-scatter', client) assert sum(counts.values()) > 0 assert all([len(r) == 1 for r in res]) finally: client.close() cluster.close() class CountSerialized(object): def __init__(self, x): self.x = x self.count = 0 def __add__(self, other): return self.x + getattr(other, 'x', other) __radd__ = __add__ def __reduce__(self): self.count += 1 return (CountSerialized, (self.x,)) def add5(a, b, c, d=0, e=0): return a + b + c + d + e def test_manual_scatter(loop): x = CountSerialized(1) y = CountSerialized(2) z = CountSerialized(3) with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as client: # noqa: F841 with parallel_config(backend='dask', scatter=[x, y]): f = delayed(add5) tasks = [f(x, y, z, d=4, e=5), f(x, z, y, d=5, e=4), f(y, x, z, d=x, e=5), f(z, z, x, d=z, e=y)] expected = [func(*args, **kwargs) for func, args, kwargs in tasks] results = Parallel()(tasks) # Scatter must take a list/tuple with pytest.raises(TypeError): with parallel_config(backend='dask', loop=loop, scatter=1): pass assert results == expected # Scattered variables only serialized once assert x.count == 1 assert y.count == 1 # Depending on the version of distributed, the unscattered z variable # is either pickled 4 or 6 times, possibly because of the memoization # of objects that appear several times in the arguments of a delayed # task. assert z.count in (4, 6) # When the same IOLoop is used for multiple clients in a row, use # loop_in_thread instead of loop to prevent the Client from closing it. See # dask/distributed #4112 def test_auto_scatter(loop_in_thread): np = pytest.importorskip('numpy') data1 = np.ones(int(1e4), dtype=np.uint8) data2 = np.ones(int(1e4), dtype=np.uint8) data_to_process = ([data1] * 3) + ([data2] * 3) with cluster() as (s, [a, b]): with Client(s['address'], loop=loop_in_thread) as client: with parallel_config(backend='dask'): # Passing the same data as arg and kwarg triggers a single # scatter operation whose result is reused. Parallel()(delayed(noop)(data, data, i, opt=data) for i, data in enumerate(data_to_process)) # By default large array are automatically scattered with # broadcast=1 which means that one worker must directly receive # the data from the scatter operation once. counts = count_events('receive-from-scatter', client) assert counts[a['address']] + counts[b['address']] == 2 with cluster() as (s, [a, b]): with Client(s['address'], loop=loop_in_thread) as client: with parallel_config(backend='dask'): Parallel()(delayed(noop)(data1[:3], i) for i in range(5)) # Small arrays are passed within the task definition without going # through a scatter operation. counts = count_events('receive-from-scatter', client) assert counts[a['address']] == 0 assert counts[b['address']] == 0 @pytest.mark.parametrize("retry_no", list(range(2))) def test_nested_scatter(loop, retry_no): np = pytest.importorskip('numpy') NUM_INNER_TASKS = 10 NUM_OUTER_TASKS = 10 def my_sum(x, i, j): return np.sum(x) def outer_function_joblib(array, i): client = get_client() # noqa with parallel_config(backend="dask"): results = Parallel()( delayed(my_sum)(array[j:], i, j) for j in range( NUM_INNER_TASKS) ) return sum(results) with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as _: with parallel_config(backend="dask"): my_array = np.ones(10000) _ = Parallel()( delayed(outer_function_joblib)( my_array[i:], i) for i in range(NUM_OUTER_TASKS) ) def test_nested_backend_context_manager(loop_in_thread): def get_nested_pids(): pids = set(Parallel(n_jobs=2)(delayed(os.getpid)() for _ in range(2))) pids |= set(Parallel(n_jobs=2)(delayed(os.getpid)() for _ in range(2))) return pids with cluster() as (s, [a, b]): with Client(s['address'], loop=loop_in_thread) as client: with parallel_config(backend='dask'): pid_groups = Parallel(n_jobs=2)( delayed(get_nested_pids)() for _ in range(10) ) for pid_group in pid_groups: assert len(set(pid_group)) <= 2 # No deadlocks with Client(s['address'], loop=loop_in_thread) as client: # noqa: F841 with parallel_config(backend='dask'): pid_groups = Parallel(n_jobs=2)( delayed(get_nested_pids)() for _ in range(10) ) for pid_group in pid_groups: assert len(set(pid_group)) <= 2 def test_nested_backend_context_manager_implicit_n_jobs(loop): # Check that Parallel with no explicit n_jobs value automatically selects # all the dask workers, including in nested calls. def _backend_type(p): return p._backend.__class__.__name__ def get_nested_implicit_n_jobs(): with Parallel() as p: return _backend_type(p), p.n_jobs with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as client: # noqa: F841 with parallel_config(backend='dask'): with Parallel() as p: assert _backend_type(p) == "DaskDistributedBackend" assert p.n_jobs == -1 all_nested_n_jobs = p( delayed(get_nested_implicit_n_jobs)() for _ in range(2) ) for backend_type, nested_n_jobs in all_nested_n_jobs: assert backend_type == "DaskDistributedBackend" assert nested_n_jobs == -1 def test_errors(loop): with pytest.raises(ValueError) as info: with parallel_config(backend='dask'): pass assert "create a dask client" in str(info.value).lower() def test_correct_nested_backend(loop): with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as client: # noqa: F841 # No requirement, should be us with parallel_config(backend='dask'): result = Parallel(n_jobs=2)( delayed(outer)(nested_require=None) for _ in range(1)) assert isinstance(result[0][0][0], DaskDistributedBackend) # Require threads, should be threading with parallel_config(backend='dask'): result = Parallel(n_jobs=2)( delayed(outer)(nested_require='sharedmem') for _ in range(1)) assert isinstance(result[0][0][0], ThreadingBackend) def outer(nested_require): return Parallel(n_jobs=2, prefer='threads')( delayed(middle)(nested_require) for _ in range(1) ) def middle(require): return Parallel(n_jobs=2, require=require)( delayed(inner)() for _ in range(1) ) def inner(): return Parallel()._backend def test_secede_with_no_processes(loop): # https://github.com/dask/distributed/issues/1775 with Client(loop=loop, processes=False, set_as_default=True): with parallel_config(backend='dask'): Parallel(n_jobs=4)(delayed(id)(i) for i in range(2)) def _worker_address(_): from distributed import get_worker return get_worker().address def test_dask_backend_keywords(loop): with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as client: # noqa: F841 with parallel_config(backend='dask', workers=a['address']): seq = Parallel()( delayed(_worker_address)(i) for i in range(10)) assert seq == [a['address']] * 10 with parallel_config(backend='dask', workers=b['address']): seq = Parallel()( delayed(_worker_address)(i) for i in range(10)) assert seq == [b['address']] * 10 def test_cleanup(loop): with Client(processes=False, loop=loop) as client: with parallel_config(backend='dask'): Parallel()(delayed(inc)(i) for i in range(10)) start = time() while client.cluster.scheduler.tasks: sleep(0.01) assert time() < start + 5 assert not client.futures @pytest.mark.parametrize("cluster_strategy", ["adaptive", "late_scaling"]) @pytest.mark.skipif( distributed.__version__ <= '2.1.1' and distributed.__version__ >= '1.28.0', reason="distributed bug - https://github.com/dask/distributed/pull/2841") def test_wait_for_workers(cluster_strategy): cluster = LocalCluster(n_workers=0, processes=False, threads_per_worker=2) client = Client(cluster) if cluster_strategy == "adaptive": cluster.adapt(minimum=0, maximum=2) elif cluster_strategy == "late_scaling": # Tell the cluster to start workers but this is a non-blocking call # and new workers might take time to connect. In this case the Parallel # call should wait for at least one worker to come up before starting # to schedule work. cluster.scale(2) try: with parallel_config(backend='dask'): # The following should wait a bit for at least one worker to # become available. Parallel()(delayed(inc)(i) for i in range(10)) finally: client.close() cluster.close() def test_wait_for_workers_timeout(): # Start a cluster with 0 worker: cluster = LocalCluster(n_workers=0, processes=False, threads_per_worker=2) client = Client(cluster) try: with parallel_config(backend='dask', wait_for_workers_timeout=0.1): # Short timeout: DaskDistributedBackend msg = "DaskDistributedBackend has no worker after 0.1 seconds." with pytest.raises(TimeoutError, match=msg): Parallel()(delayed(inc)(i) for i in range(10)) with parallel_config(backend='dask', wait_for_workers_timeout=0): # No timeout: fallback to generic joblib failure: msg = "DaskDistributedBackend has no active worker" with pytest.raises(RuntimeError, match=msg): Parallel()(delayed(inc)(i) for i in range(10)) finally: client.close() cluster.close() @pytest.mark.parametrize("backend", ["loky", "multiprocessing"]) def test_joblib_warning_inside_dask_daemonic_worker(backend): cluster = LocalCluster(n_workers=2) client = Client(cluster) def func_using_joblib_parallel(): # Somehow trying to check the warning type here (e.g. with # pytest.warns(UserWarning)) make the test hang. Work-around: return # the warning record to the client and the warning check is done # client-side. with warnings.catch_warnings(record=True) as record: Parallel(n_jobs=2, backend=backend)( delayed(inc)(i) for i in range(10)) return record fut = client.submit(func_using_joblib_parallel) record = fut.result() assert len(record) == 1 warning = record[0].message assert isinstance(warning, UserWarning) assert "distributed.worker.daemon" in str(warning) joblib-1.3.2/joblib/test/test_disk.py000066400000000000000000000042351446465525000175600ustar00rootroot00000000000000""" Unit tests for the disk utilities. """ # Authors: Gael Varoquaux # Lars Buitinck # Copyright (c) 2010 Gael Varoquaux # License: BSD Style, 3 clauses. from __future__ import with_statement import array import os from joblib.disk import disk_used, memstr_to_bytes, mkdirp, rm_subdirs from joblib.testing import parametrize, raises ############################################################################### def test_disk_used(tmpdir): cachedir = tmpdir.strpath # Not write a file that is 1M big in this directory, and check the # size. The reason we use such a big file is that it makes us robust # to errors due to block allocation. a = array.array('i') sizeof_i = a.itemsize target_size = 1024 n = int(target_size * 1024 / sizeof_i) a = array.array('i', n * (1,)) with open(os.path.join(cachedir, 'test'), 'wb') as output: a.tofile(output) assert disk_used(cachedir) >= target_size assert disk_used(cachedir) < target_size + 12 @parametrize('text,value', [('80G', 80 * 1024 ** 3), ('1.4M', int(1.4 * 1024 ** 2)), ('120M', 120 * 1024 ** 2), ('53K', 53 * 1024)]) def test_memstr_to_bytes(text, value): assert memstr_to_bytes(text) == value @parametrize('text,exception,regex', [('fooG', ValueError, r'Invalid literal for size.*fooG.*'), ('1.4N', ValueError, r'Invalid literal for size.*1.4N.*')]) def test_memstr_to_bytes_exception(text, exception, regex): with raises(exception) as excinfo: memstr_to_bytes(text) assert excinfo.match(regex) def test_mkdirp(tmpdir): mkdirp(os.path.join(tmpdir.strpath, 'ham')) mkdirp(os.path.join(tmpdir.strpath, 'ham')) mkdirp(os.path.join(tmpdir.strpath, 'spam', 'spam')) # Not all OSErrors are ignored with raises(OSError): mkdirp('') def test_rm_subdirs(tmpdir): sub_path = os.path.join(tmpdir.strpath, "am", "stram") full_path = os.path.join(sub_path, "gram") mkdirp(os.path.join(full_path)) rm_subdirs(sub_path) assert os.path.exists(sub_path) assert not os.path.exists(full_path) joblib-1.3.2/joblib/test/test_func_inspect.py000066400000000000000000000224201446465525000213020ustar00rootroot00000000000000""" Test the func_inspect module. """ # Author: Gael Varoquaux # Copyright (c) 2009 Gael Varoquaux # License: BSD Style, 3 clauses. import functools from joblib.func_inspect import filter_args, get_func_name, get_func_code from joblib.func_inspect import _clean_win_chars, format_signature from joblib.memory import Memory from joblib.test.common import with_numpy from joblib.testing import fixture, parametrize, raises ############################################################################### # Module-level functions and fixture, for tests def f(x, y=0): pass def g(x): pass def h(x, y=0, *args, **kwargs): pass def i(x=1): pass def j(x, y, **kwargs): pass def k(*args, **kwargs): pass def m1(x, *, y): pass def m2(x, *, y, z=3): pass @fixture(scope='module') def cached_func(tmpdir_factory): # Create a Memory object to test decorated functions. # We should be careful not to call the decorated functions, so that # cache directories are not created in the temp dir. cachedir = tmpdir_factory.mktemp("joblib_test_func_inspect") mem = Memory(cachedir.strpath) @mem.cache def cached_func_inner(x): return x return cached_func_inner class Klass(object): def f(self, x): return x ############################################################################### # Tests @parametrize('func,args,filtered_args', [(f, [[], (1, )], {'x': 1, 'y': 0}), (f, [['x'], (1, )], {'y': 0}), (f, [['y'], (0, )], {'x': 0}), (f, [['y'], (0, ), {'y': 1}], {'x': 0}), (f, [['x', 'y'], (0, )], {}), (f, [[], (0,), {'y': 1}], {'x': 0, 'y': 1}), (f, [['y'], (), {'x': 2, 'y': 1}], {'x': 2}), (g, [[], (), {'x': 1}], {'x': 1}), (i, [[], (2, )], {'x': 2})]) def test_filter_args(func, args, filtered_args): assert filter_args(func, *args) == filtered_args def test_filter_args_method(): obj = Klass() assert filter_args(obj.f, [], (1, )) == {'x': 1, 'self': obj} @parametrize('func,args,filtered_args', [(h, [[], (1, )], {'x': 1, 'y': 0, '*': [], '**': {}}), (h, [[], (1, 2, 3, 4)], {'x': 1, 'y': 2, '*': [3, 4], '**': {}}), (h, [[], (1, 25), {'ee': 2}], {'x': 1, 'y': 25, '*': [], '**': {'ee': 2}}), (h, [['*'], (1, 2, 25), {'ee': 2}], {'x': 1, 'y': 2, '**': {'ee': 2}})]) def test_filter_varargs(func, args, filtered_args): assert filter_args(func, *args) == filtered_args test_filter_kwargs_extra_params = [ (m1, [[], (1,), {'y': 2}], {'x': 1, 'y': 2}), (m2, [[], (1,), {'y': 2}], {'x': 1, 'y': 2, 'z': 3}) ] @parametrize('func,args,filtered_args', [(k, [[], (1, 2), {'ee': 2}], {'*': [1, 2], '**': {'ee': 2}}), (k, [[], (3, 4)], {'*': [3, 4], '**': {}})] + test_filter_kwargs_extra_params) def test_filter_kwargs(func, args, filtered_args): assert filter_args(func, *args) == filtered_args def test_filter_args_2(): assert (filter_args(j, [], (1, 2), {'ee': 2}) == {'x': 1, 'y': 2, '**': {'ee': 2}}) ff = functools.partial(f, 1) # filter_args has to special-case partial assert filter_args(ff, [], (1, )) == {'*': [1], '**': {}} assert filter_args(ff, ['y'], (1, )) == {'*': [1], '**': {}} @parametrize('func,funcname', [(f, 'f'), (g, 'g'), (cached_func, 'cached_func')]) def test_func_name(func, funcname): # Check that we are not confused by decoration # here testcase 'cached_func' is the function itself assert get_func_name(func)[1] == funcname def test_func_name_on_inner_func(cached_func): # Check that we are not confused by decoration # here testcase 'cached_func' is the 'cached_func_inner' function # returned by 'cached_func' fixture assert get_func_name(cached_func)[1] == 'cached_func_inner' def test_func_name_collision_on_inner_func(): # Check that two functions defining and caching an inner function # with the same do not cause (module, name) collision def f(): def inner_func(): return # pragma: no cover return get_func_name(inner_func) def g(): def inner_func(): return # pragma: no cover return get_func_name(inner_func) module, name = f() other_module, other_name = g() assert name == other_name assert module != other_module def test_func_inspect_errors(): # Check that func_inspect is robust and will work on weird objects assert get_func_name('a'.lower)[-1] == 'lower' assert get_func_code('a'.lower)[1:] == (None, -1) ff = lambda x: x # noqa: E731 assert get_func_name(ff, win_characters=False)[-1] == '' assert get_func_code(ff)[1] == __file__.replace('.pyc', '.py') # Simulate a function defined in __main__ ff.__module__ = '__main__' assert get_func_name(ff, win_characters=False)[-1] == '' assert get_func_code(ff)[1] == __file__.replace('.pyc', '.py') def func_with_kwonly_args(a, b, *, kw1='kw1', kw2='kw2'): pass def func_with_signature(a: int, b: int) -> None: pass def test_filter_args_edge_cases(): assert ( filter_args(func_with_kwonly_args, [], (1, 2), {'kw1': 3, 'kw2': 4}) == {'a': 1, 'b': 2, 'kw1': 3, 'kw2': 4}) # filter_args doesn't care about keyword-only arguments so you # can pass 'kw1' into *args without any problem with raises(ValueError) as excinfo: filter_args(func_with_kwonly_args, [], (1, 2, 3), {'kw2': 2}) excinfo.match("Keyword-only parameter 'kw1' was passed as positional " "parameter") assert ( filter_args(func_with_kwonly_args, ['b', 'kw2'], (1, 2), {'kw1': 3, 'kw2': 4}) == {'a': 1, 'kw1': 3}) assert (filter_args(func_with_signature, ['b'], (1, 2)) == {'a': 1}) def test_bound_methods(): """ Make sure that calling the same method on two different instances of the same class does resolv to different signatures. """ a = Klass() b = Klass() assert filter_args(a.f, [], (1, )) != filter_args(b.f, [], (1, )) @parametrize('exception,regex,func,args', [(ValueError, 'ignore_lst must be a list of parameters to ignore', f, ['bar', (None, )]), (ValueError, r'Ignore list: argument \'(.*)\' is not defined', g, [['bar'], (None, )]), (ValueError, 'Wrong number of arguments', h, [[]])]) def test_filter_args_error_msg(exception, regex, func, args): """ Make sure that filter_args returns decent error messages, for the sake of the user. """ with raises(exception) as excinfo: filter_args(func, *args) excinfo.match(regex) def test_filter_args_no_kwargs_mutation(): """None-regression test against 0.12.0 changes. https://github.com/joblib/joblib/pull/75 Make sure filter args doesn't mutate the kwargs dict that gets passed in. """ kwargs = {'x': 0} filter_args(g, [], [], kwargs) assert kwargs == {'x': 0} def test_clean_win_chars(): string = r'C:\foo\bar\main.py' mangled_string = _clean_win_chars(string) for char in ('\\', ':', '<', '>', '!'): assert char not in mangled_string @parametrize('func,args,kwargs,sgn_expected', [(g, [list(range(5))], {}, 'g([0, 1, 2, 3, 4])'), (k, [1, 2, (3, 4)], {'y': True}, 'k(1, 2, (3, 4), y=True)')]) def test_format_signature(func, args, kwargs, sgn_expected): # Test signature formatting. path, sgn_result = format_signature(func, *args, **kwargs) assert sgn_result == sgn_expected def test_format_signature_long_arguments(): shortening_threshold = 1500 # shortening gets it down to 700 characters but there is the name # of the function in the signature and a few additional things # like dots for the ellipsis shortening_target = 700 + 10 arg = 'a' * shortening_threshold _, signature = format_signature(h, arg) assert len(signature) < shortening_target nb_args = 5 args = [arg for _ in range(nb_args)] _, signature = format_signature(h, *args) assert len(signature) < shortening_target * nb_args kwargs = {str(i): arg for i, arg in enumerate(args)} _, signature = format_signature(h, **kwargs) assert len(signature) < shortening_target * nb_args _, signature = format_signature(h, *args, **kwargs) assert len(signature) < shortening_target * 2 * nb_args @with_numpy def test_format_signature_numpy(): """ Test the format signature formatting with numpy. """ def test_special_source_encoding(): from joblib.test.test_func_inspect_special_encoding import big5_f func_code, source_file, first_line = get_func_code(big5_f) assert first_line == 5 assert "def big5_f():" in func_code assert "test_func_inspect_special_encoding" in source_file def _get_code(): from joblib.test.test_func_inspect_special_encoding import big5_f return get_func_code(big5_f)[0] def test_func_code_consistency(): from joblib.parallel import Parallel, delayed codes = Parallel(n_jobs=2)(delayed(_get_code)() for _ in range(5)) assert len(set(codes)) == 1 joblib-1.3.2/joblib/test/test_func_inspect_special_encoding.py000066400000000000000000000002211446465525000246430ustar00rootroot00000000000000# -*- coding: big5 -*- # Some Traditional Chinese characters: @Ǥr def big5_f(): """Ωժ """ # return 0 joblib-1.3.2/joblib/test/test_hashing.py000066400000000000000000000372661446465525000202610ustar00rootroot00000000000000""" Test the hashing module. """ # Author: Gael Varoquaux # Copyright (c) 2009 Gael Varoquaux # License: BSD Style, 3 clauses. import time import hashlib import sys import gc import io import collections import itertools import pickle import random from concurrent.futures import ProcessPoolExecutor from decimal import Decimal from joblib.hashing import hash from joblib.func_inspect import filter_args from joblib.memory import Memory from joblib.testing import raises, skipif, fixture, parametrize from joblib.test.common import np, with_numpy def unicode(s): return s ############################################################################### # Helper functions for the tests def time_func(func, *args): """ Time function func on *args. """ times = list() for _ in range(3): t1 = time.time() func(*args) times.append(time.time() - t1) return min(times) def relative_time(func1, func2, *args): """ Return the relative time between func1 and func2 applied on *args. """ time_func1 = time_func(func1, *args) time_func2 = time_func(func2, *args) relative_diff = 0.5 * (abs(time_func1 - time_func2) / (time_func1 + time_func2)) return relative_diff class Klass(object): def f(self, x): return x class KlassWithCachedMethod(object): def __init__(self, cachedir): mem = Memory(location=cachedir) self.f = mem.cache(self.f) def f(self, x): return x ############################################################################### # Tests input_list = [1, 2, 1., 2., 1 + 1j, 2. + 1j, 'a', 'b', (1,), (1, 1,), [1, ], [1, 1, ], {1: 1}, {1: 2}, {2: 1}, None, gc.collect, [1, ].append, # Next 2 sets have unorderable elements in python 3. set(('a', 1)), set(('a', 1, ('a', 1))), # Next 2 dicts have unorderable type of keys in python 3. {'a': 1, 1: 2}, {'a': 1, 1: 2, 'd': {'a': 1}}] @parametrize('obj1', input_list) @parametrize('obj2', input_list) def test_trivial_hash(obj1, obj2): """Smoke test hash on various types.""" # Check that 2 objects have the same hash only if they are the same. are_hashes_equal = hash(obj1) == hash(obj2) are_objs_identical = obj1 is obj2 assert are_hashes_equal == are_objs_identical def test_hash_methods(): # Check that hashing instance methods works a = io.StringIO(unicode('a')) assert hash(a.flush) == hash(a.flush) a1 = collections.deque(range(10)) a2 = collections.deque(range(9)) assert hash(a1.extend) != hash(a2.extend) @fixture(scope='function') @with_numpy def three_np_arrays(): rnd = np.random.RandomState(0) arr1 = rnd.random_sample((10, 10)) arr2 = arr1.copy() arr3 = arr2.copy() arr3[0] += 1 return arr1, arr2, arr3 def test_hash_numpy_arrays(three_np_arrays): arr1, arr2, arr3 = three_np_arrays for obj1, obj2 in itertools.product(three_np_arrays, repeat=2): are_hashes_equal = hash(obj1) == hash(obj2) are_arrays_equal = np.all(obj1 == obj2) assert are_hashes_equal == are_arrays_equal assert hash(arr1) != hash(arr1.T) def test_hash_numpy_dict_of_arrays(three_np_arrays): arr1, arr2, arr3 = three_np_arrays d1 = {1: arr1, 2: arr2} d2 = {1: arr2, 2: arr1} d3 = {1: arr2, 2: arr3} assert hash(d1) == hash(d2) assert hash(d1) != hash(d3) @with_numpy @parametrize('dtype', ['datetime64[s]', 'timedelta64[D]']) def test_numpy_datetime_array(dtype): # memoryview is not supported for some dtypes e.g. datetime64 # see https://github.com/joblib/joblib/issues/188 for more details a_hash = hash(np.arange(10)) array = np.arange(0, 10, dtype=dtype) assert hash(array) != a_hash @with_numpy def test_hash_numpy_noncontiguous(): a = np.asarray(np.arange(6000).reshape((1000, 2, 3)), order='F')[:, :1, :] b = np.ascontiguousarray(a) assert hash(a) != hash(b) c = np.asfortranarray(a) assert hash(a) != hash(c) @with_numpy @parametrize('coerce_mmap', [True, False]) def test_hash_memmap(tmpdir, coerce_mmap): """Check that memmap and arrays hash identically if coerce_mmap is True.""" filename = tmpdir.join('memmap_temp').strpath try: m = np.memmap(filename, shape=(10, 10), mode='w+') a = np.asarray(m) are_hashes_equal = (hash(a, coerce_mmap=coerce_mmap) == hash(m, coerce_mmap=coerce_mmap)) assert are_hashes_equal == coerce_mmap finally: if 'm' in locals(): del m # Force a garbage-collection cycle, to be certain that the # object is delete, and we don't run in a problem under # Windows with a file handle still open. gc.collect() @with_numpy @skipif(sys.platform == 'win32', reason='This test is not stable under windows' ' for some reason') def test_hash_numpy_performance(): """ Check the performance of hashing numpy arrays: In [22]: a = np.random.random(1000000) In [23]: %timeit hashlib.md5(a).hexdigest() 100 loops, best of 3: 20.7 ms per loop In [24]: %timeit hashlib.md5(pickle.dumps(a, protocol=2)).hexdigest() 1 loops, best of 3: 73.1 ms per loop In [25]: %timeit hashlib.md5(cPickle.dumps(a, protocol=2)).hexdigest() 10 loops, best of 3: 53.9 ms per loop In [26]: %timeit hash(a) 100 loops, best of 3: 20.8 ms per loop """ rnd = np.random.RandomState(0) a = rnd.random_sample(1000000) def md5_hash(x): return hashlib.md5(memoryview(x)).hexdigest() relative_diff = relative_time(md5_hash, hash, a) assert relative_diff < 0.3 # Check that hashing an tuple of 3 arrays takes approximately # 3 times as much as hashing one array time_hashlib = 3 * time_func(md5_hash, a) time_hash = time_func(hash, (a, a, a)) relative_diff = 0.5 * (abs(time_hash - time_hashlib) / (time_hash + time_hashlib)) assert relative_diff < 0.3 def test_bound_methods_hash(): """ Make sure that calling the same method on two different instances of the same class does resolve to the same hashes. """ a = Klass() b = Klass() assert (hash(filter_args(a.f, [], (1, ))) == hash(filter_args(b.f, [], (1, )))) def test_bound_cached_methods_hash(tmpdir): """ Make sure that calling the same _cached_ method on two different instances of the same class does resolve to the same hashes. """ a = KlassWithCachedMethod(tmpdir.strpath) b = KlassWithCachedMethod(tmpdir.strpath) assert (hash(filter_args(a.f.func, [], (1, ))) == hash(filter_args(b.f.func, [], (1, )))) @with_numpy def test_hash_object_dtype(): """ Make sure that ndarrays with dtype `object' hash correctly.""" a = np.array([np.arange(i) for i in range(6)], dtype=object) b = np.array([np.arange(i) for i in range(6)], dtype=object) assert hash(a) == hash(b) @with_numpy def test_numpy_scalar(): # Numpy scalars are built from compiled functions, and lead to # strange pickling paths explored, that can give hash collisions a = np.float64(2.0) b = np.float64(3.0) assert hash(a) != hash(b) def test_dict_hash(tmpdir): # Check that dictionaries hash consistently, even though the ordering # of the keys is not guaranteed k = KlassWithCachedMethod(tmpdir.strpath) d = {'#s12069__c_maps.nii.gz': [33], '#s12158__c_maps.nii.gz': [33], '#s12258__c_maps.nii.gz': [33], '#s12277__c_maps.nii.gz': [33], '#s12300__c_maps.nii.gz': [33], '#s12401__c_maps.nii.gz': [33], '#s12430__c_maps.nii.gz': [33], '#s13817__c_maps.nii.gz': [33], '#s13903__c_maps.nii.gz': [33], '#s13916__c_maps.nii.gz': [33], '#s13981__c_maps.nii.gz': [33], '#s13982__c_maps.nii.gz': [33], '#s13983__c_maps.nii.gz': [33]} a = k.f(d) b = k.f(a) assert hash(a) == hash(b) def test_set_hash(tmpdir): # Check that sets hash consistently, even though their ordering # is not guaranteed k = KlassWithCachedMethod(tmpdir.strpath) s = set(['#s12069__c_maps.nii.gz', '#s12158__c_maps.nii.gz', '#s12258__c_maps.nii.gz', '#s12277__c_maps.nii.gz', '#s12300__c_maps.nii.gz', '#s12401__c_maps.nii.gz', '#s12430__c_maps.nii.gz', '#s13817__c_maps.nii.gz', '#s13903__c_maps.nii.gz', '#s13916__c_maps.nii.gz', '#s13981__c_maps.nii.gz', '#s13982__c_maps.nii.gz', '#s13983__c_maps.nii.gz']) a = k.f(s) b = k.f(a) assert hash(a) == hash(b) def test_set_decimal_hash(): # Check that sets containing decimals hash consistently, even though # ordering is not guaranteed assert (hash(set([Decimal(0), Decimal('NaN')])) == hash(set([Decimal('NaN'), Decimal(0)]))) def test_string(): # Test that we obtain the same hash for object owning several strings, # whatever the past of these strings (which are immutable in Python) string = 'foo' a = {string: 'bar'} b = {string: 'bar'} c = pickle.loads(pickle.dumps(b)) assert hash([a, b]) == hash([a, c]) @with_numpy def test_numpy_dtype_pickling(): # numpy dtype hashing is tricky to get right: see #231, #239, #251 #1080, # #1082, and explanatory comments inside # ``joblib.hashing.NumpyHasher.save``. # In this test, we make sure that the pickling of numpy dtypes is robust to # object identity and object copy. dt1 = np.dtype('f4') dt2 = np.dtype('f4') # simple dtypes objects are interned assert dt1 is dt2 assert hash(dt1) == hash(dt2) dt1_roundtripped = pickle.loads(pickle.dumps(dt1)) assert dt1 is not dt1_roundtripped assert hash(dt1) == hash(dt1_roundtripped) assert hash([dt1, dt1]) == hash([dt1_roundtripped, dt1_roundtripped]) assert hash([dt1, dt1]) == hash([dt1, dt1_roundtripped]) complex_dt1 = np.dtype( [('name', np.str_, 16), ('grades', np.float64, (2,))] ) complex_dt2 = np.dtype( [('name', np.str_, 16), ('grades', np.float64, (2,))] ) # complex dtypes objects are not interned assert hash(complex_dt1) == hash(complex_dt2) complex_dt1_roundtripped = pickle.loads(pickle.dumps(complex_dt1)) assert complex_dt1_roundtripped is not complex_dt1 assert hash(complex_dt1) == hash(complex_dt1_roundtripped) assert hash([complex_dt1, complex_dt1]) == hash( [complex_dt1_roundtripped, complex_dt1_roundtripped] ) assert hash([complex_dt1, complex_dt1]) == hash( [complex_dt1_roundtripped, complex_dt1] ) @parametrize('to_hash,expected', [('This is a string to hash', '71b3f47df22cb19431d85d92d0b230b2'), (u"C'est l\xe9t\xe9", '2d8d189e9b2b0b2e384d93c868c0e576'), ((123456, 54321, -98765), 'e205227dd82250871fa25aa0ec690aa3'), ([random.Random(42).random() for _ in range(5)], 'a11ffad81f9682a7d901e6edc3d16c84'), ({'abcde': 123, 'sadfas': [-9999, 2, 3]}, 'aeda150553d4bb5c69f0e69d51b0e2ef')]) def test_hashes_stay_the_same(to_hash, expected): # We want to make sure that hashes don't change with joblib # version. For end users, that would mean that they have to # regenerate their cache from scratch, which potentially means # lengthy recomputations. # Expected results have been generated with joblib 0.9.2 assert hash(to_hash) == expected @with_numpy def test_hashes_are_different_between_c_and_fortran_contiguous_arrays(): # We want to be sure that the c-contiguous and f-contiguous versions of the # same array produce 2 different hashes. rng = np.random.RandomState(0) arr_c = rng.random_sample((10, 10)) arr_f = np.asfortranarray(arr_c) assert hash(arr_c) != hash(arr_f) @with_numpy def test_0d_array(): hash(np.array(0)) @with_numpy def test_0d_and_1d_array_hashing_is_different(): assert hash(np.array(0)) != hash(np.array([0])) @with_numpy def test_hashes_stay_the_same_with_numpy_objects(): # Note: joblib used to test numpy objects hashing by comparing the produced # hash of an object with some hard-coded target value to guarantee that # hashing remains the same across joblib versions. However, since numpy # 1.20 and joblib 1.0, joblib relies on potentially unstable implementation # details of numpy to hash np.dtype objects, which makes the stability of # hash values across different environments hard to guarantee and to test. # As a result, hashing stability across joblib versions becomes best-effort # only, and we only test the consistency within a single environment by # making sure: # - the hash of two copies of the same objects is the same # - hashing some object in two different python processes produces the same # value. This should be viewed as a proxy for testing hash consistency # through time between Python sessions (provided no change in the # environment was done between sessions). def create_objects_to_hash(): rng = np.random.RandomState(42) # Being explicit about dtypes in order to avoid # architecture-related differences. Also using 'f4' rather than # 'f8' for float arrays because 'f8' arrays generated by # rng.random.randn don't seem to be bit-identical on 32bit and # 64bit machines. to_hash_list = [ rng.randint(-1000, high=1000, size=50).astype(' # Copyright (c) 2009 Gael Varoquaux # License: BSD Style, 3 clauses. import re from joblib.logger import PrintTime def test_print_time(tmpdir, capsys): # A simple smoke test for PrintTime. logfile = tmpdir.join('test.log').strpath print_time = PrintTime(logfile=logfile) print_time('Foo') # Create a second time, to smoke test log rotation. print_time = PrintTime(logfile=logfile) print_time('Foo') # And a third time print_time = PrintTime(logfile=logfile) print_time('Foo') out_printed_text, err_printed_text = capsys.readouterr() # Use regexps to be robust to time variations match = r"Foo: 0\..s, 0\..min\nFoo: 0\..s, 0..min\nFoo: " + \ r".\..s, 0..min\n" if not re.match(match, err_printed_text): raise AssertionError('Excepted %s, got %s' % (match, err_printed_text)) joblib-1.3.2/joblib/test/test_memmapping.py000066400000000000000000001244421446465525000207630ustar00rootroot00000000000000import os import mmap import sys import platform import gc import pickle import itertools from time import sleep import subprocess import threading import faulthandler import pytest from joblib.test.common import with_numpy, np from joblib.test.common import with_multiprocessing from joblib.test.common import with_dev_shm from joblib.testing import raises, parametrize, skipif from joblib.backports import make_memmap from joblib.parallel import Parallel, delayed from joblib.pool import MemmappingPool from joblib.executor import _TestingMemmappingExecutor as TestExecutor from joblib._memmapping_reducer import has_shareable_memory from joblib._memmapping_reducer import ArrayMemmapForwardReducer from joblib._memmapping_reducer import _strided_from_memmap from joblib._memmapping_reducer import _get_temp_dir from joblib._memmapping_reducer import _WeakArrayKeyMap from joblib._memmapping_reducer import _get_backing_memmap import joblib._memmapping_reducer as jmr def setup_module(): faulthandler.dump_traceback_later(timeout=300, exit=True) def teardown_module(): faulthandler.cancel_dump_traceback_later() def check_memmap_and_send_back(array): assert _get_backing_memmap(array) is not None return array def check_array(args): """Dummy helper function to be executed in subprocesses Check that the provided array has the expected values in the provided range. """ data, position, expected = args np.testing.assert_array_equal(data[position], expected) def inplace_double(args): """Dummy helper function to be executed in subprocesses Check that the input array has the right values in the provided range and perform an inplace modification to double the values in the range by two. """ data, position, expected = args assert data[position] == expected data[position] *= 2 np.testing.assert_array_equal(data[position], 2 * expected) @with_numpy @with_multiprocessing def test_memmap_based_array_reducing(tmpdir): """Check that it is possible to reduce a memmap backed array""" assert_array_equal = np.testing.assert_array_equal filename = tmpdir.join('test.mmap').strpath # Create a file larger than what will be used by a buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+') # Fill the original buffer with negative markers to detect over of # underflow in case of test failures buffer[:] = - 1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype) buffer.flush() # Memmap a 2D fortran array on a offsetted subsection of the previous # buffer a = np.memmap(filename, dtype=np.float64, shape=(3, 5, 4), mode='r+', order='F', offset=4) a[:] = np.arange(60).reshape(a.shape) # Build various views that share the buffer with the original memmap # b is an memmap sliced view on an memmap instance b = a[1:-1, 2:-1, 2:4] # c and d are array views c = np.asarray(b) d = c.T # Array reducer with auto dumping disabled reducer = ArrayMemmapForwardReducer(None, tmpdir.strpath, 'c', True) def reconstruct_array_or_memmap(x): cons, args = reducer(x) return cons(*args) # Reconstruct original memmap a_reconstructed = reconstruct_array_or_memmap(a) assert has_shareable_memory(a_reconstructed) assert isinstance(a_reconstructed, np.memmap) assert_array_equal(a_reconstructed, a) # Reconstruct strided memmap view b_reconstructed = reconstruct_array_or_memmap(b) assert has_shareable_memory(b_reconstructed) assert_array_equal(b_reconstructed, b) # Reconstruct arrays views on memmap base c_reconstructed = reconstruct_array_or_memmap(c) assert not isinstance(c_reconstructed, np.memmap) assert has_shareable_memory(c_reconstructed) assert_array_equal(c_reconstructed, c) d_reconstructed = reconstruct_array_or_memmap(d) assert not isinstance(d_reconstructed, np.memmap) assert has_shareable_memory(d_reconstructed) assert_array_equal(d_reconstructed, d) # Test graceful degradation on fake memmap instances with in-memory # buffers a3 = a * 3 assert not has_shareable_memory(a3) a3_reconstructed = reconstruct_array_or_memmap(a3) assert not has_shareable_memory(a3_reconstructed) assert not isinstance(a3_reconstructed, np.memmap) assert_array_equal(a3_reconstructed, a * 3) # Test graceful degradation on arrays derived from fake memmap instances b3 = np.asarray(a3) assert not has_shareable_memory(b3) b3_reconstructed = reconstruct_array_or_memmap(b3) assert isinstance(b3_reconstructed, np.ndarray) assert not has_shareable_memory(b3_reconstructed) assert_array_equal(b3_reconstructed, b3) @with_multiprocessing @skipif((sys.platform != "win32") or (), reason="PermissionError only easily triggerable on Windows") def test_resource_tracker_retries_when_permissionerror(tmpdir): # Test resource_tracker retry mechanism when unlinking memmaps. See more # thorough information in the ``unlink_file`` documentation of joblib. filename = tmpdir.join('test.mmap').strpath cmd = """if 1: import os import numpy as np import time from joblib.externals.loky.backend import resource_tracker resource_tracker.VERBOSE = 1 # Start the resource tracker resource_tracker.ensure_running() time.sleep(1) # Create a file containing numpy data memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+') memmap[:] = np.arange(10).astype(np.int8).data memmap.flush() assert os.path.exists(r"{filename}") del memmap # Create a np.memmap backed by this file memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+') resource_tracker.register(r"{filename}", "file") # Ask the resource_tracker to delete the file backing the np.memmap , this # should raise PermissionError that the resource_tracker will log. resource_tracker.maybe_unlink(r"{filename}", "file") # Wait for the resource_tracker to process the maybe_unlink before cleaning # up the memmap time.sleep(2) """.format(filename=filename) p = subprocess.Popen([sys.executable, '-c', cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE) p.wait() out, err = p.communicate() assert p.returncode == 0 assert out == b'' msg = 'tried to unlink {}, got PermissionError'.format(filename) assert msg in err.decode() @with_numpy @with_multiprocessing def test_high_dimension_memmap_array_reducing(tmpdir): assert_array_equal = np.testing.assert_array_equal filename = tmpdir.join('test.mmap').strpath # Create a high dimensional memmap a = np.memmap(filename, dtype=np.float64, shape=(100, 15, 15, 3), mode='w+') a[:] = np.arange(100 * 15 * 15 * 3).reshape(a.shape) # Create some slices/indices at various dimensions b = a[0:10] c = a[:, 5:10] d = a[:, :, :, 0] e = a[1:3:4] # Array reducer with auto dumping disabled reducer = ArrayMemmapForwardReducer(None, tmpdir.strpath, 'c', True) def reconstruct_array_or_memmap(x): cons, args = reducer(x) return cons(*args) a_reconstructed = reconstruct_array_or_memmap(a) assert has_shareable_memory(a_reconstructed) assert isinstance(a_reconstructed, np.memmap) assert_array_equal(a_reconstructed, a) b_reconstructed = reconstruct_array_or_memmap(b) assert has_shareable_memory(b_reconstructed) assert_array_equal(b_reconstructed, b) c_reconstructed = reconstruct_array_or_memmap(c) assert has_shareable_memory(c_reconstructed) assert_array_equal(c_reconstructed, c) d_reconstructed = reconstruct_array_or_memmap(d) assert has_shareable_memory(d_reconstructed) assert_array_equal(d_reconstructed, d) e_reconstructed = reconstruct_array_or_memmap(e) assert has_shareable_memory(e_reconstructed) assert_array_equal(e_reconstructed, e) @with_numpy def test__strided_from_memmap(tmpdir): fname = tmpdir.join('test.mmap').strpath size = 5 * mmap.ALLOCATIONGRANULARITY offset = mmap.ALLOCATIONGRANULARITY + 1 # This line creates the mmap file that is reused later memmap_obj = np.memmap(fname, mode='w+', shape=size + offset) # filename, dtype, mode, offset, order, shape, strides, total_buffer_len memmap_obj = _strided_from_memmap(fname, dtype='uint8', mode='r', offset=offset, order='C', shape=size, strides=None, total_buffer_len=None, unlink_on_gc_collect=False) assert isinstance(memmap_obj, np.memmap) assert memmap_obj.offset == offset memmap_backed_obj = _strided_from_memmap( fname, dtype='uint8', mode='r', offset=offset, order='C', shape=(size // 2,), strides=(2,), total_buffer_len=size, unlink_on_gc_collect=False ) assert _get_backing_memmap(memmap_backed_obj).offset == offset @with_numpy @with_multiprocessing @parametrize("factory", [MemmappingPool, TestExecutor.get_memmapping_executor], ids=["multiprocessing", "loky"]) def test_pool_with_memmap(factory, tmpdir): """Check that subprocess can access and update shared memory memmap""" assert_array_equal = np.testing.assert_array_equal # Fork the subprocess before allocating the objects to be passed pool_temp_folder = tmpdir.mkdir('pool').strpath p = factory(10, max_nbytes=2, temp_folder=pool_temp_folder) try: filename = tmpdir.join('test.mmap').strpath a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+') a.fill(1.0) p.map(inplace_double, [(a, (i, j), 1.0) for i in range(a.shape[0]) for j in range(a.shape[1])]) assert_array_equal(a, 2 * np.ones(a.shape)) # Open a copy-on-write view on the previous data b = np.memmap(filename, dtype=np.float32, shape=(5, 3), mode='c') p.map(inplace_double, [(b, (i, j), 2.0) for i in range(b.shape[0]) for j in range(b.shape[1])]) # Passing memmap instances to the pool should not trigger the creation # of new files on the FS assert os.listdir(pool_temp_folder) == [] # the original data is untouched assert_array_equal(a, 2 * np.ones(a.shape)) assert_array_equal(b, 2 * np.ones(b.shape)) # readonly maps can be read but not updated c = np.memmap(filename, dtype=np.float32, shape=(10,), mode='r', offset=5 * 4) with raises(AssertionError): p.map(check_array, [(c, i, 3.0) for i in range(c.shape[0])]) # depending on the version of numpy one can either get a RuntimeError # or a ValueError with raises((RuntimeError, ValueError)): p.map(inplace_double, [(c, i, 2.0) for i in range(c.shape[0])]) finally: # Clean all filehandlers held by the pool p.terminate() del p @with_numpy @with_multiprocessing @parametrize("factory", [MemmappingPool, TestExecutor.get_memmapping_executor], ids=["multiprocessing", "loky"]) def test_pool_with_memmap_array_view(factory, tmpdir): """Check that subprocess can access and update shared memory array""" assert_array_equal = np.testing.assert_array_equal # Fork the subprocess before allocating the objects to be passed pool_temp_folder = tmpdir.mkdir('pool').strpath p = factory(10, max_nbytes=2, temp_folder=pool_temp_folder) try: filename = tmpdir.join('test.mmap').strpath a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+') a.fill(1.0) # Create an ndarray view on the memmap instance a_view = np.asarray(a) assert not isinstance(a_view, np.memmap) assert has_shareable_memory(a_view) p.map(inplace_double, [(a_view, (i, j), 1.0) for i in range(a.shape[0]) for j in range(a.shape[1])]) # Both a and the a_view have been updated assert_array_equal(a, 2 * np.ones(a.shape)) assert_array_equal(a_view, 2 * np.ones(a.shape)) # Passing memmap array view to the pool should not trigger the # creation of new files on the FS assert os.listdir(pool_temp_folder) == [] finally: p.terminate() del p @with_numpy @with_multiprocessing @parametrize("backend", ["multiprocessing", "loky"]) def test_permission_error_windows_reference_cycle(backend): # Non regression test for: # https://github.com/joblib/joblib/issues/806 # # The issue happens when trying to delete a memory mapped file that has # not yet been closed by one of the worker processes. cmd = """if 1: import numpy as np from joblib import Parallel, delayed data = np.random.rand(int(2e6)).reshape((int(1e6), 2)) # Build a complex cyclic reference that is likely to delay garbage # collection of the memmapped array in the worker processes. first_list = current_list = [data] for i in range(10): current_list = [current_list] first_list.append(current_list) if __name__ == "__main__": results = Parallel(n_jobs=2, backend="{b}")( delayed(len)(current_list) for i in range(10)) assert results == [1] * 10 """.format(b=backend) p = subprocess.Popen([sys.executable, '-c', cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE) p.wait() out, err = p.communicate() assert p.returncode == 0, out.decode() + "\n\n" + err.decode() @with_numpy @with_multiprocessing @parametrize("backend", ["multiprocessing", "loky"]) def test_permission_error_windows_memmap_sent_to_parent(backend): # Second non-regression test for: # https://github.com/joblib/joblib/issues/806 # previously, child process would not convert temporary memmaps to numpy # arrays when sending the data back to the parent process. This would lead # to permission errors on windows when deleting joblib's temporary folder, # as the memmaped files handles would still opened in the parent process. cmd = '''if 1: import os import time import numpy as np from joblib import Parallel, delayed from testutils import return_slice_of_data data = np.ones(int(2e6)) if __name__ == '__main__': # warm-up call to launch the workers and start the resource_tracker _ = Parallel(n_jobs=2, verbose=5, backend='{b}')( delayed(id)(i) for i in range(20)) time.sleep(0.5) slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')( delayed(return_slice_of_data)(data, 0, 20) for _ in range(10)) '''.format(b=backend) for _ in range(3): env = os.environ.copy() env['PYTHONPATH'] = os.path.dirname(__file__) p = subprocess.Popen([sys.executable, '-c', cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env) p.wait() out, err = p.communicate() assert p.returncode == 0, err assert out == b'' if sys.version_info[:3] not in [(3, 8, 0), (3, 8, 1)]: # In early versions of Python 3.8, a reference leak # https://github.com/cloudpipe/cloudpickle/issues/327, holds # references to pickled objects, generating race condition during # cleanup finalizers of joblib and noisy resource_tracker outputs. assert b'resource_tracker' not in err @with_numpy @with_multiprocessing @parametrize("backend", ["multiprocessing", "loky"]) def test_parallel_isolated_temp_folders(backend): # Test that consecutive Parallel call use isolated subfolders, even # for the loky backend that reuses its executor instance across calls. array = np.arange(int(1e2)) [filename_1] = Parallel(n_jobs=2, backend=backend, max_nbytes=10)( delayed(getattr)(array, 'filename') for _ in range(1) ) [filename_2] = Parallel(n_jobs=2, backend=backend, max_nbytes=10)( delayed(getattr)(array, 'filename') for _ in range(1) ) assert os.path.dirname(filename_2) != os.path.dirname(filename_1) @with_numpy @with_multiprocessing @parametrize("backend", ["multiprocessing", "loky"]) def test_managed_backend_reuse_temp_folder(backend): # Test that calls to a managed parallel object reuse the same memmaps. array = np.arange(int(1e2)) with Parallel(n_jobs=2, backend=backend, max_nbytes=10) as p: [filename_1] = p( delayed(getattr)(array, 'filename') for _ in range(1) ) [filename_2] = p( delayed(getattr)(array, 'filename') for _ in range(1) ) assert os.path.dirname(filename_2) == os.path.dirname(filename_1) @with_numpy @with_multiprocessing def test_memmapping_temp_folder_thread_safety(): # Concurrent calls to Parallel with the loky backend will use the same # executor, and thus the same reducers. Make sure that those reducers use # different temporary folders depending on which Parallel objects called # them, which is necessary to limit potential race conditions during the # garbage collection of temporary memmaps. array = np.arange(int(1e2)) temp_dirs_thread_1 = set() temp_dirs_thread_2 = set() def concurrent_get_filename(array, temp_dirs): with Parallel(backend='loky', n_jobs=2, max_nbytes=10) as p: for i in range(10): [filename] = p( delayed(getattr)(array, 'filename') for _ in range(1) ) temp_dirs.add(os.path.dirname(filename)) t1 = threading.Thread( target=concurrent_get_filename, args=(array, temp_dirs_thread_1) ) t2 = threading.Thread( target=concurrent_get_filename, args=(array, temp_dirs_thread_2) ) t1.start() t2.start() t1.join() t2.join() assert len(temp_dirs_thread_1) == 1 assert len(temp_dirs_thread_2) == 1 assert temp_dirs_thread_1 != temp_dirs_thread_2 @with_numpy @with_multiprocessing def test_multithreaded_parallel_termination_resource_tracker_silent(): # test that concurrent termination attempts of a same executor does not # emit any spurious error from the resource_tracker. We test various # situations making 0, 1 or both parallel call sending a task that will # make the worker (and thus the whole Parallel call) error out. cmd = '''if 1: import os import numpy as np from joblib import Parallel, delayed from joblib.externals.loky.backend import resource_tracker from concurrent.futures import ThreadPoolExecutor, wait resource_tracker.VERBOSE = 0 array = np.arange(int(1e2)) temp_dirs_thread_1 = set() temp_dirs_thread_2 = set() def raise_error(array): raise ValueError def parallel_get_filename(array, temp_dirs): with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p: for i in range(10): [filename] = p( delayed(getattr)(array, "filename") for _ in range(1) ) temp_dirs.add(os.path.dirname(filename)) def parallel_raise(array, temp_dirs): with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p: for i in range(10): [filename] = p( delayed(raise_error)(array) for _ in range(1) ) temp_dirs.add(os.path.dirname(filename)) executor = ThreadPoolExecutor(max_workers=2) # both function calls will use the same loky executor, but with a # different Parallel object. future_1 = executor.submit({f1}, array, temp_dirs_thread_1) future_2 = executor.submit({f2}, array, temp_dirs_thread_2) # Wait for both threads to terminate their backend wait([future_1, future_2]) future_1.result() future_2.result() ''' functions_and_returncodes = [ ("parallel_get_filename", "parallel_get_filename", 0), ("parallel_get_filename", "parallel_raise", 1), ("parallel_raise", "parallel_raise", 1) ] for f1, f2, returncode in functions_and_returncodes: p = subprocess.Popen([sys.executable, '-c', cmd.format(f1=f1, f2=f2)], stderr=subprocess.PIPE, stdout=subprocess.PIPE) p.wait() out, err = p.communicate() assert p.returncode == returncode, out.decode() assert b"resource_tracker" not in err, err.decode() @with_numpy @with_multiprocessing @parametrize("backend", ["multiprocessing", "loky"]) def test_many_parallel_calls_on_same_object(backend): # After #966 got merged, consecutive Parallel objects were sharing temp # folder, which would lead to race conditions happening during the # temporary resources management with the resource_tracker. This is a # non-regression test that makes sure that consecutive Parallel operations # on the same object do not error out. cmd = '''if 1: import os import time import numpy as np from joblib import Parallel, delayed from testutils import return_slice_of_data data = np.ones(100) if __name__ == '__main__': for i in range(5): slice_of_data = Parallel( n_jobs=2, max_nbytes=1, backend='{b}')( delayed(return_slice_of_data)(data, 0, 20) for _ in range(10) ) '''.format(b=backend) env = os.environ.copy() env['PYTHONPATH'] = os.path.dirname(__file__) p = subprocess.Popen( [sys.executable, '-c', cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env, ) p.wait() out, err = p.communicate() assert p.returncode == 0, err assert out == b'' if sys.version_info[:3] not in [(3, 8, 0), (3, 8, 1)]: # In early versions of Python 3.8, a reference leak # https://github.com/cloudpipe/cloudpickle/issues/327, holds # references to pickled objects, generating race condition during # cleanup finalizers of joblib and noisy resource_tracker outputs. assert b'resource_tracker' not in err @with_numpy @with_multiprocessing @parametrize("backend", ["multiprocessing", "loky"]) def test_memmap_returned_as_regular_array(backend): data = np.ones(int(1e3)) # Check that child processes send temporary memmaps back as numpy arrays. [result] = Parallel(n_jobs=2, backend=backend, max_nbytes=100)( delayed(check_memmap_and_send_back)(data) for _ in range(1)) assert _get_backing_memmap(result) is None @with_numpy @with_multiprocessing @parametrize("backend", ["multiprocessing", "loky"]) def test_resource_tracker_silent_when_reference_cycles(backend): # There is a variety of reasons that can make joblib with loky backend # output noisy warnings when a reference cycle is preventing a memmap from # being garbage collected. Especially, joblib's main process finalizer # deletes the temporary folder if it was not done before, which can # interact badly with the resource_tracker. We don't risk leaking any # resources, but this will likely make joblib output a lot of low-level # confusing messages. # # This test makes sure that the resource_tracker is silent when a reference # has been collected concurrently on non-Windows platforms. # # Note that the script in ``cmd`` is the exact same script as in # test_permission_error_windows_reference_cycle. if backend == "loky" and sys.platform.startswith('win'): # XXX: on Windows, reference cycles can delay timely garbage collection # and make it impossible to properly delete the temporary folder in the # main process because of permission errors. pytest.xfail( "The temporary folder cannot be deleted on Windows in the " "presence of a reference cycle" ) cmd = """if 1: import numpy as np from joblib import Parallel, delayed data = np.random.rand(int(2e6)).reshape((int(1e6), 2)) # Build a complex cyclic reference that is likely to delay garbage # collection of the memmapped array in the worker processes. first_list = current_list = [data] for i in range(10): current_list = [current_list] first_list.append(current_list) if __name__ == "__main__": results = Parallel(n_jobs=2, backend="{b}")( delayed(len)(current_list) for i in range(10)) assert results == [1] * 10 """.format(b=backend) p = subprocess.Popen([sys.executable, '-c', cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE) p.wait() out, err = p.communicate() out = out.decode() err = err.decode() assert p.returncode == 0, out + "\n\n" + err assert "resource_tracker" not in err, err @with_numpy @with_multiprocessing @parametrize("factory", [MemmappingPool, TestExecutor.get_memmapping_executor], ids=["multiprocessing", "loky"]) def test_memmapping_pool_for_large_arrays(factory, tmpdir): """Check that large arrays are not copied in memory""" # Check that the tempfolder is empty assert os.listdir(tmpdir.strpath) == [] # Build an array reducers that automatically dump large array content # to filesystem backed memmap instances to avoid memory explosion p = factory(3, max_nbytes=40, temp_folder=tmpdir.strpath, verbose=2) try: # The temporary folder for the pool is not provisioned in advance assert os.listdir(tmpdir.strpath) == [] assert not os.path.exists(p._temp_folder) small = np.ones(5, dtype=np.float32) assert small.nbytes == 20 p.map(check_array, [(small, i, 1.0) for i in range(small.shape[0])]) # Memory has been copied, the pool filesystem folder is unused assert os.listdir(tmpdir.strpath) == [] # Try with a file larger than the memmap threshold of 40 bytes large = np.ones(100, dtype=np.float64) assert large.nbytes == 800 p.map(check_array, [(large, i, 1.0) for i in range(large.shape[0])]) # The data has been dumped in a temp folder for subprocess to share it # without per-child memory copies assert os.path.isdir(p._temp_folder) dumped_filenames = os.listdir(p._temp_folder) assert len(dumped_filenames) == 1 # Check that memory mapping is not triggered for arrays with # dtype='object' objects = np.array(['abc'] * 100, dtype='object') results = p.map(has_shareable_memory, [objects]) assert not results[0] finally: # check FS garbage upon pool termination p.terminate() for i in range(10): sleep(.1) if not os.path.exists(p._temp_folder): break else: # pragma: no cover raise AssertionError( 'temporary folder {} was not deleted'.format(p._temp_folder) ) del p @with_numpy @with_multiprocessing @parametrize( "backend", [ pytest.param( "multiprocessing", marks=pytest.mark.xfail( reason='https://github.com/joblib/joblib/issues/1086' ), ), "loky", ] ) def test_child_raises_parent_exits_cleanly(backend): # When a task executed by a child process raises an error, the parent # process's backend is notified, and calls abort_everything. # In loky, abort_everything itself calls shutdown(kill_workers=True) which # sends SIGKILL to the worker, preventing it from running the finalizers # supposed to signal the resource_tracker when the worker is done using # objects relying on a shared resource (e.g np.memmaps). Because this # behavior is prone to : # - cause a resource leak # - make the resource tracker emit noisy resource warnings # we explicitly test that, when the said situation occurs: # - no resources are actually leaked # - the temporary resources are deleted as soon as possible (typically, at # the end of the failing Parallel call) # - the resource_tracker does not emit any warnings. cmd = """if 1: import os from pathlib import Path from time import sleep import numpy as np from joblib import Parallel, delayed from testutils import print_filename_and_raise data = np.random.rand(1000) def get_temp_folder(parallel_obj, backend): if "{b}" == "loky": return Path(parallel_obj._backend._workers._temp_folder) else: return Path(parallel_obj._backend._pool._temp_folder) if __name__ == "__main__": try: with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p: temp_folder = get_temp_folder(p, "{b}") p(delayed(print_filename_and_raise)(data) for i in range(1)) except ValueError as e: # the temporary folder should be deleted by the end of this # call but apparently on some file systems, this takes # some time to be visible. # # We attempt to write into the temporary folder to test for # its existence and we wait for a maximum of 10 seconds. for i in range(100): try: with open(temp_folder / "some_file.txt", "w") as f: f.write("some content") except FileNotFoundError: # temp_folder has been deleted, all is fine break # ... else, wait a bit and try again sleep(.1) else: raise AssertionError( str(temp_folder) + " was not deleted" ) from e """.format(b=backend) env = os.environ.copy() env['PYTHONPATH'] = os.path.dirname(__file__) p = subprocess.Popen([sys.executable, '-c', cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env) p.wait() out, err = p.communicate() out, err = out.decode(), err.decode() filename = out.split('\n')[0] assert p.returncode == 0, err or out assert err == '' # no resource_tracker warnings. assert not os.path.exists(filename) @with_numpy @with_multiprocessing @parametrize("factory", [MemmappingPool, TestExecutor.get_memmapping_executor], ids=["multiprocessing", "loky"]) def test_memmapping_pool_for_large_arrays_disabled(factory, tmpdir): """Check that large arrays memmapping can be disabled""" # Set max_nbytes to None to disable the auto memmapping feature p = factory(3, max_nbytes=None, temp_folder=tmpdir.strpath) try: # Check that the tempfolder is empty assert os.listdir(tmpdir.strpath) == [] # Try with a file largish than the memmap threshold of 40 bytes large = np.ones(100, dtype=np.float64) assert large.nbytes == 800 p.map(check_array, [(large, i, 1.0) for i in range(large.shape[0])]) # Check that the tempfolder is still empty assert os.listdir(tmpdir.strpath) == [] finally: # Cleanup open file descriptors p.terminate() del p @with_numpy @with_multiprocessing @with_dev_shm @parametrize("factory", [MemmappingPool, TestExecutor.get_memmapping_executor], ids=["multiprocessing", "loky"]) def test_memmapping_on_large_enough_dev_shm(factory): """Check that memmapping uses /dev/shm when possible""" orig_size = jmr.SYSTEM_SHARED_MEM_FS_MIN_SIZE try: # Make joblib believe that it can use /dev/shm even when running on a # CI container where the size of the /dev/shm is not very large (that # is at least 32 MB instead of 2 GB by default). jmr.SYSTEM_SHARED_MEM_FS_MIN_SIZE = int(32e6) p = factory(3, max_nbytes=10) try: # Check that the pool has correctly detected the presence of the # shared memory filesystem. pool_temp_folder = p._temp_folder folder_prefix = '/dev/shm/joblib_memmapping_folder_' assert pool_temp_folder.startswith(folder_prefix) assert os.path.exists(pool_temp_folder) # Try with a file larger than the memmap threshold of 10 bytes a = np.ones(100, dtype=np.float64) assert a.nbytes == 800 p.map(id, [a] * 10) # a should have been memmapped to the pool temp folder: the joblib # pickling procedure generate one .pkl file: assert len(os.listdir(pool_temp_folder)) == 1 # create a new array with content that is different from 'a' so # that it is mapped to a different file in the temporary folder of # the pool. b = np.ones(100, dtype=np.float64) * 2 assert b.nbytes == 800 p.map(id, [b] * 10) # A copy of both a and b are now stored in the shared memory folder assert len(os.listdir(pool_temp_folder)) == 2 finally: # Cleanup open file descriptors p.terminate() del p for i in range(100): # The temp folder is cleaned up upon pool termination if not os.path.exists(pool_temp_folder): break sleep(.1) else: # pragma: no cover raise AssertionError('temporary folder of pool was not deleted') finally: jmr.SYSTEM_SHARED_MEM_FS_MIN_SIZE = orig_size @with_numpy @with_multiprocessing @with_dev_shm @parametrize("factory", [MemmappingPool, TestExecutor.get_memmapping_executor], ids=["multiprocessing", "loky"]) def test_memmapping_on_too_small_dev_shm(factory): orig_size = jmr.SYSTEM_SHARED_MEM_FS_MIN_SIZE try: # Make joblib believe that it cannot use /dev/shm unless there is # 42 exabytes of available shared memory in /dev/shm jmr.SYSTEM_SHARED_MEM_FS_MIN_SIZE = int(42e18) p = factory(3, max_nbytes=10) try: # Check that the pool has correctly detected the presence of the # shared memory filesystem. pool_temp_folder = p._temp_folder assert not pool_temp_folder.startswith('/dev/shm') finally: # Cleanup open file descriptors p.terminate() del p # The temp folder is cleaned up upon pool termination assert not os.path.exists(pool_temp_folder) finally: jmr.SYSTEM_SHARED_MEM_FS_MIN_SIZE = orig_size @with_numpy @with_multiprocessing @parametrize("factory", [MemmappingPool, TestExecutor.get_memmapping_executor], ids=["multiprocessing", "loky"]) def test_memmapping_pool_for_large_arrays_in_return(factory, tmpdir): """Check that large arrays are not copied in memory in return""" assert_array_equal = np.testing.assert_array_equal # Build an array reducers that automatically dump large array content # but check that the returned datastructure are regular arrays to avoid # passing a memmap array pointing to a pool controlled temp folder that # might be confusing to the user # The MemmappingPool user can always return numpy.memmap object explicitly # to avoid memory copy p = factory(3, max_nbytes=10, temp_folder=tmpdir.strpath) try: res = p.apply_async(np.ones, args=(1000,)) large = res.get() assert not has_shareable_memory(large) assert_array_equal(large, np.ones(1000)) finally: p.terminate() del p def _worker_multiply(a, n_times): """Multiplication function to be executed by subprocess""" assert has_shareable_memory(a) return a * n_times @with_numpy @with_multiprocessing @parametrize("factory", [MemmappingPool, TestExecutor.get_memmapping_executor], ids=["multiprocessing", "loky"]) def test_workaround_against_bad_memmap_with_copied_buffers(factory, tmpdir): """Check that memmaps with a bad buffer are returned as regular arrays Unary operations and ufuncs on memmap instances return a new memmap instance with an in-memory buffer (probably a numpy bug). """ assert_array_equal = np.testing.assert_array_equal p = factory(3, max_nbytes=10, temp_folder=tmpdir.strpath) try: # Send a complex, large-ish view on a array that will be converted to # a memmap in the worker process a = np.asarray(np.arange(6000).reshape((1000, 2, 3)), order='F')[:, :1, :] # Call a non-inplace multiply operation on the worker and memmap and # send it back to the parent. b = p.apply_async(_worker_multiply, args=(a, 3)).get() assert not has_shareable_memory(b) assert_array_equal(b, 3 * a) finally: p.terminate() del p def identity(arg): return arg @with_numpy @with_multiprocessing @parametrize( "factory,retry_no", list(itertools.product( [MemmappingPool, TestExecutor.get_memmapping_executor], range(3))), ids=['{}, {}'.format(x, y) for x, y in itertools.product( ["multiprocessing", "loky"], map(str, range(3)))]) def test_pool_memmap_with_big_offset(factory, retry_no, tmpdir): # Test that numpy memmap offset is set correctly if greater than # mmap.ALLOCATIONGRANULARITY, see # https://github.com/joblib/joblib/issues/451 and # https://github.com/numpy/numpy/pull/8443 for more details. fname = tmpdir.join('test.mmap').strpath size = 5 * mmap.ALLOCATIONGRANULARITY offset = mmap.ALLOCATIONGRANULARITY + 1 obj = make_memmap(fname, mode='w+', shape=size, dtype='uint8', offset=offset) p = factory(2, temp_folder=tmpdir.strpath) result = p.apply_async(identity, args=(obj,)).get() assert isinstance(result, np.memmap) assert result.offset == offset np.testing.assert_array_equal(obj, result) p.terminate() def test_pool_get_temp_dir(tmpdir): pool_folder_name = 'test.tmpdir' pool_folder, shared_mem = _get_temp_dir(pool_folder_name, tmpdir.strpath) assert shared_mem is False assert pool_folder == tmpdir.join('test.tmpdir').strpath pool_folder, shared_mem = _get_temp_dir(pool_folder_name, temp_folder=None) if sys.platform.startswith('win'): assert shared_mem is False assert pool_folder.endswith(pool_folder_name) def test_pool_get_temp_dir_no_statvfs(tmpdir, monkeypatch): """Check that _get_temp_dir works when os.statvfs is not defined Regression test for #902 """ pool_folder_name = 'test.tmpdir' import joblib._memmapping_reducer if hasattr(joblib._memmapping_reducer.os, 'statvfs'): # We are on Unix, since Windows doesn't have this function monkeypatch.delattr(joblib._memmapping_reducer.os, 'statvfs') pool_folder, shared_mem = _get_temp_dir(pool_folder_name, temp_folder=None) if sys.platform.startswith('win'): assert shared_mem is False assert pool_folder.endswith(pool_folder_name) @with_numpy @skipif(sys.platform == 'win32', reason='This test fails with a ' 'PermissionError on Windows') @parametrize("mmap_mode", ["r+", "w+"]) def test_numpy_arrays_use_different_memory(mmap_mode): def func(arr, value): arr[:] = value return arr arrays = [np.zeros((10, 10), dtype='float64') for i in range(10)] results = Parallel(mmap_mode=mmap_mode, max_nbytes=0, n_jobs=2)( delayed(func)(arr, i) for i, arr in enumerate(arrays)) for i, arr in enumerate(results): np.testing.assert_array_equal(arr, i) @with_numpy def test_weak_array_key_map(): def assert_empty_after_gc_collect(container, retries=100): for i in range(retries): if len(container) == 0: return gc.collect() sleep(.1) assert len(container) == 0 a = np.ones(42) m = _WeakArrayKeyMap() m.set(a, 'a') assert m.get(a) == 'a' b = a assert m.get(b) == 'a' m.set(b, 'b') assert m.get(a) == 'b' del a gc.collect() assert len(m._data) == 1 assert m.get(b) == 'b' del b assert_empty_after_gc_collect(m._data) c = np.ones(42) m.set(c, 'c') assert len(m._data) == 1 assert m.get(c) == 'c' with raises(KeyError): m.get(np.ones(42)) del c assert_empty_after_gc_collect(m._data) # Check that creating and dropping numpy arrays with potentially the same # object id will not cause the map to get confused. def get_set_get_collect(m, i): a = np.ones(42) with raises(KeyError): m.get(a) m.set(a, i) assert m.get(a) == i return id(a) unique_ids = set([get_set_get_collect(m, i) for i in range(1000)]) if platform.python_implementation() == 'CPython': # On CPython (at least) the same id is often reused many times for the # temporary arrays created under the local scope of the # get_set_get_collect function without causing any spurious lookups / # insertions in the map. Apparently on Python nogil, the id is not # reused as often. max_len_unique_ids = 400 if getattr(sys.flags, 'nogil', False) else 100 assert len(unique_ids) < max_len_unique_ids def test_weak_array_key_map_no_pickling(): m = _WeakArrayKeyMap() with raises(pickle.PicklingError): pickle.dumps(m) @with_numpy @with_multiprocessing def test_direct_mmap(tmpdir): testfile = str(tmpdir.join('arr.dat')) a = np.arange(10, dtype='uint8') a.tofile(testfile) def _read_array(): with open(testfile) as fd: mm = mmap.mmap(fd.fileno(), 0, access=mmap.ACCESS_READ, offset=0) return np.ndarray((10,), dtype=np.uint8, buffer=mm, offset=0) def func(x): return x**2 arr = _read_array() # this is expected to work and gives the reference ref = Parallel(n_jobs=2)(delayed(func)(x) for x in [a]) # now test that it work with the mmap array results = Parallel(n_jobs=2)(delayed(func)(x) for x in [arr]) np.testing.assert_array_equal(results, ref) # also test with a mmap array read in the subprocess def worker(): return _read_array() results = Parallel(n_jobs=2)(delayed(worker)() for _ in range(1)) np.testing.assert_array_equal(results[0], arr) joblib-1.3.2/joblib/test/test_memory.py000066400000000000000000001402351446465525000201370ustar00rootroot00000000000000""" Test the memory module. """ # Author: Gael Varoquaux # Copyright (c) 2009 Gael Varoquaux # License: BSD Style, 3 clauses. import functools import gc import logging import shutil import os import os.path import pathlib import pickle import sys import time import datetime import textwrap import pytest from joblib.memory import Memory from joblib.memory import expires_after from joblib.memory import MemorizedFunc, NotMemorizedFunc from joblib.memory import MemorizedResult, NotMemorizedResult from joblib.memory import _FUNCTION_HASHES from joblib.memory import register_store_backend, _STORE_BACKENDS from joblib.memory import _build_func_identifier, _store_backend_factory from joblib.memory import JobLibCollisionWarning from joblib.parallel import Parallel, delayed from joblib._store_backends import StoreBackendBase, FileSystemStoreBackend from joblib.test.common import with_numpy, np from joblib.test.common import with_multiprocessing from joblib.testing import parametrize, raises, warns from joblib.hashing import hash ############################################################################### # Module-level variables for the tests def f(x, y=1): """ A module-level function for testing purposes. """ return x ** 2 + y ############################################################################### # Helper function for the tests def check_identity_lazy(func, accumulator, location): """ Given a function and an accumulator (a list that grows every time the function is called), check that the function can be decorated by memory to be a lazy identity. """ # Call each function with several arguments, and check that it is # evaluated only once per argument. memory = Memory(location=location, verbose=0) func = memory.cache(func) for i in range(3): for _ in range(2): assert func(i) == i assert len(accumulator) == i + 1 def corrupt_single_cache_item(memory): single_cache_item, = memory.store_backend.get_items() output_filename = os.path.join(single_cache_item.path, 'output.pkl') with open(output_filename, 'w') as f: f.write('garbage') def monkeypatch_cached_func_warn(func, monkeypatch_fixture): # Need monkeypatch because pytest does not # capture stdlib logging output (see # https://github.com/pytest-dev/pytest/issues/2079) recorded = [] def append_to_record(item): recorded.append(item) monkeypatch_fixture.setattr(func, 'warn', append_to_record) return recorded ############################################################################### # Tests def test_memory_integration(tmpdir): """ Simple test of memory lazy evaluation. """ accumulator = list() # Rmk: this function has the same name than a module-level function, # thus it serves as a test to see that both are identified # as different. def f(arg): accumulator.append(1) return arg check_identity_lazy(f, accumulator, tmpdir.strpath) # Now test clearing for compress in (False, True): for mmap_mode in ('r', None): memory = Memory(location=tmpdir.strpath, verbose=10, mmap_mode=mmap_mode, compress=compress) # First clear the cache directory, to check that our code can # handle that # NOTE: this line would raise an exception, as the database file is # still open; we ignore the error since we want to test what # happens if the directory disappears shutil.rmtree(tmpdir.strpath, ignore_errors=True) g = memory.cache(f) g(1) g.clear(warn=False) current_accumulator = len(accumulator) out = g(1) assert len(accumulator) == current_accumulator + 1 # Also, check that Memory.eval works similarly assert memory.eval(f, 1) == out assert len(accumulator) == current_accumulator + 1 # Now do a smoke test with a function defined in __main__, as the name # mangling rules are more complex f.__module__ = '__main__' memory = Memory(location=tmpdir.strpath, verbose=0) memory.cache(f)(1) @parametrize("call_before_reducing", [True, False]) def test_parallel_call_cached_function_defined_in_jupyter( tmpdir, call_before_reducing ): # Calling an interactively defined memory.cache()'d function inside a # Parallel call used to clear the existing cache related to the said # function (https://github.com/joblib/joblib/issues/1035) # This tests checks that this is no longer the case. # TODO: test that the cache related to the function cache persists across # ipython sessions (provided that no code change were made to the # function's source)? # The first part of the test makes the necessary low-level calls to emulate # the definition of a function in an jupyter notebook cell. Joblib has # some custom code to treat functions defined specifically in jupyter # notebooks/ipython session -- we want to test this code, which requires # the emulation to be rigorous. for session_no in [0, 1]: ipython_cell_source = ''' def f(x): return x ''' ipython_cell_id = ''.format(session_no) exec( compile( textwrap.dedent(ipython_cell_source), filename=ipython_cell_id, mode='exec' ) ) # f is now accessible in the locals mapping - but for some unknown # reason, f = locals()['f'] throws a KeyError at runtime, we need to # bind locals()['f'] to a different name in the local namespace aliased_f = locals()['f'] aliased_f.__module__ = "__main__" # Preliminary sanity checks, and tests checking that joblib properly # identified f as an interactive function defined in a jupyter notebook assert aliased_f(1) == 1 assert aliased_f.__code__.co_filename == ipython_cell_id memory = Memory(location=tmpdir.strpath, verbose=0) cached_f = memory.cache(aliased_f) assert len(os.listdir(tmpdir / 'joblib')) == 1 f_cache_relative_directory = os.listdir(tmpdir / 'joblib')[0] assert 'ipython-input' in f_cache_relative_directory f_cache_directory = tmpdir / 'joblib' / f_cache_relative_directory if session_no == 0: # The cache should be empty as cached_f has not been called yet. assert os.listdir(f_cache_directory) == ['f'] assert os.listdir(f_cache_directory / 'f') == [] if call_before_reducing: cached_f(3) # Two files were just created, func_code.py, and a folder # containing the information (inputs hash/ouptput) of # cached_f(3) assert len(os.listdir(f_cache_directory / 'f')) == 2 # Now, testing #1035: when calling a cached function, joblib # used to dynamically inspect the underlying function to # extract its source code (to verify it matches the source code # of the function as last inspected by joblib) -- however, # source code introspection fails for dynamic functions sent to # child processes - which would eventually make joblib clear # the cache associated to f res = Parallel(n_jobs=2)(delayed(cached_f)(i) for i in [1, 2]) else: # Submit the function to the joblib child processes, although # the function has never been called in the parent yet. This # triggers a specific code branch inside # MemorizedFunc.__reduce__. res = Parallel(n_jobs=2)(delayed(cached_f)(i) for i in [1, 2]) assert len(os.listdir(f_cache_directory / 'f')) == 3 cached_f(3) # Making sure f's cache does not get cleared after the parallel # calls, and contains ALL cached functions calls (f(1), f(2), f(3)) # and 'func_code.py' assert len(os.listdir(f_cache_directory / 'f')) == 4 else: # For the second session, there should be an already existing cache assert len(os.listdir(f_cache_directory / 'f')) == 4 cached_f(3) # The previous cache should not be invalidated after calling the # function in a new session assert len(os.listdir(f_cache_directory / 'f')) == 4 def test_no_memory(): """ Test memory with location=None: no memoize """ accumulator = list() def ff(arg): accumulator.append(1) return arg memory = Memory(location=None, verbose=0) gg = memory.cache(ff) for _ in range(4): current_accumulator = len(accumulator) gg(1) assert len(accumulator) == current_accumulator + 1 def test_memory_kwarg(tmpdir): " Test memory with a function with keyword arguments." accumulator = list() def g(arg1=None, arg2=1): accumulator.append(1) return arg1 check_identity_lazy(g, accumulator, tmpdir.strpath) memory = Memory(location=tmpdir.strpath, verbose=0) g = memory.cache(g) # Smoke test with an explicit keyword argument: assert g(arg1=30, arg2=2) == 30 def test_memory_lambda(tmpdir): " Test memory with a function with a lambda." accumulator = list() def helper(x): """ A helper function to define l as a lambda. """ accumulator.append(1) return x check_identity_lazy(lambda x: helper(x), accumulator, tmpdir.strpath) def test_memory_name_collision(tmpdir): " Check that name collisions with functions will raise warnings" memory = Memory(location=tmpdir.strpath, verbose=0) @memory.cache def name_collision(x): """ A first function called name_collision """ return x a = name_collision @memory.cache def name_collision(x): """ A second function called name_collision """ return x b = name_collision with warns(JobLibCollisionWarning) as warninfo: a(1) b(1) assert len(warninfo) == 1 assert "collision" in str(warninfo[0].message) def test_memory_warning_lambda_collisions(tmpdir): # Check that multiple use of lambda will raise collisions memory = Memory(location=tmpdir.strpath, verbose=0) a = memory.cache(lambda x: x) b = memory.cache(lambda x: x + 1) with warns(JobLibCollisionWarning) as warninfo: assert a(0) == 0 assert b(1) == 2 assert a(1) == 1 # In recent Python versions, we can retrieve the code of lambdas, # thus nothing is raised assert len(warninfo) == 4 def test_memory_warning_collision_detection(tmpdir): # Check that collisions impossible to detect will raise appropriate # warnings. memory = Memory(location=tmpdir.strpath, verbose=0) a1 = eval('lambda x: x') a1 = memory.cache(a1) b1 = eval('lambda x: x+1') b1 = memory.cache(b1) with warns(JobLibCollisionWarning) as warninfo: a1(1) b1(1) a1(0) assert len(warninfo) == 2 assert "cannot detect" in str(warninfo[0].message).lower() def test_memory_partial(tmpdir): " Test memory with functools.partial." accumulator = list() def func(x, y): """ A helper function to define l as a lambda. """ accumulator.append(1) return y import functools function = functools.partial(func, 1) check_identity_lazy(function, accumulator, tmpdir.strpath) def test_memory_eval(tmpdir): " Smoke test memory with a function with a function defined in an eval." memory = Memory(location=tmpdir.strpath, verbose=0) m = eval('lambda x: x') mm = memory.cache(m) assert mm(1) == 1 def count_and_append(x=[]): """ A function with a side effect in its arguments. Return the length of its argument and append one element. """ len_x = len(x) x.append(None) return len_x def test_argument_change(tmpdir): """ Check that if a function has a side effect in its arguments, it should use the hash of changing arguments. """ memory = Memory(location=tmpdir.strpath, verbose=0) func = memory.cache(count_and_append) # call the function for the first time, is should cache it with # argument x=[] assert func() == 0 # the second time the argument is x=[None], which is not cached # yet, so the functions should be called a second time assert func() == 1 @with_numpy @parametrize('mmap_mode', [None, 'r']) def test_memory_numpy(tmpdir, mmap_mode): " Test memory with a function with numpy arrays." accumulator = list() def n(arg=None): accumulator.append(1) return arg memory = Memory(location=tmpdir.strpath, mmap_mode=mmap_mode, verbose=0) cached_n = memory.cache(n) rnd = np.random.RandomState(0) for i in range(3): a = rnd.random_sample((10, 10)) for _ in range(3): assert np.all(cached_n(a) == a) assert len(accumulator) == i + 1 @with_numpy def test_memory_numpy_check_mmap_mode(tmpdir, monkeypatch): """Check that mmap_mode is respected even at the first call""" memory = Memory(location=tmpdir.strpath, mmap_mode='r', verbose=0) @memory.cache() def twice(a): return a * 2 a = np.ones(3) b = twice(a) c = twice(a) assert isinstance(c, np.memmap) assert c.mode == 'r' assert isinstance(b, np.memmap) assert b.mode == 'r' # Corrupts the file, Deleting b and c mmaps # is necessary to be able edit the file del b del c gc.collect() corrupt_single_cache_item(memory) # Make sure that corrupting the file causes recomputation and that # a warning is issued. recorded_warnings = monkeypatch_cached_func_warn(twice, monkeypatch) d = twice(a) assert len(recorded_warnings) == 1 exception_msg = 'Exception while loading results' assert exception_msg in recorded_warnings[0] # Asserts that the recomputation returns a mmap assert isinstance(d, np.memmap) assert d.mode == 'r' def test_memory_exception(tmpdir): """ Smoketest the exception handling of Memory. """ memory = Memory(location=tmpdir.strpath, verbose=0) class MyException(Exception): pass @memory.cache def h(exc=0): if exc: raise MyException # Call once, to initialise the cache h() for _ in range(3): # Call 3 times, to be sure that the Exception is always raised with raises(MyException): h(1) def test_memory_ignore(tmpdir): " Test the ignore feature of memory " memory = Memory(location=tmpdir.strpath, verbose=0) accumulator = list() @memory.cache(ignore=['y']) def z(x, y=1): accumulator.append(1) assert z.ignore == ['y'] z(0, y=1) assert len(accumulator) == 1 z(0, y=1) assert len(accumulator) == 1 z(0, y=2) assert len(accumulator) == 1 def test_memory_ignore_decorated(tmpdir): " Test the ignore feature of memory on a decorated function " memory = Memory(location=tmpdir.strpath, verbose=0) accumulator = list() def decorate(f): @functools.wraps(f) def wrapped(*args, **kwargs): return f(*args, **kwargs) return wrapped @memory.cache(ignore=['y']) @decorate def z(x, y=1): accumulator.append(1) assert z.ignore == ['y'] z(0, y=1) assert len(accumulator) == 1 z(0, y=1) assert len(accumulator) == 1 z(0, y=2) assert len(accumulator) == 1 def test_memory_args_as_kwargs(tmpdir): """Non-regression test against 0.12.0 changes. https://github.com/joblib/joblib/pull/751 """ memory = Memory(location=tmpdir.strpath, verbose=0) @memory.cache def plus_one(a): return a + 1 # It's possible to call a positional arg as a kwarg. assert plus_one(1) == 2 assert plus_one(a=1) == 2 # However, a positional argument that joblib hadn't seen # before would cause a failure if it was passed as a kwarg. assert plus_one(a=2) == 3 @parametrize('ignore, verbose, mmap_mode', [(['x'], 100, 'r'), ([], 10, None)]) def test_partial_decoration(tmpdir, ignore, verbose, mmap_mode): "Check cache may be called with kwargs before decorating" memory = Memory(location=tmpdir.strpath, verbose=0) @memory.cache(ignore=ignore, verbose=verbose, mmap_mode=mmap_mode) def z(x): pass assert z.ignore == ignore assert z._verbose == verbose assert z.mmap_mode == mmap_mode def test_func_dir(tmpdir): # Test the creation of the memory cache directory for the function. memory = Memory(location=tmpdir.strpath, verbose=0) path = __name__.split('.') path.append('f') path = tmpdir.join('joblib', *path).strpath g = memory.cache(f) # Test that the function directory is created on demand func_id = _build_func_identifier(f) location = os.path.join(g.store_backend.location, func_id) assert location == path assert os.path.exists(path) assert memory.location == os.path.dirname(g.store_backend.location) # Test that the code is stored. # For the following test to be robust to previous execution, we clear # the in-memory store _FUNCTION_HASHES.clear() assert not g._check_previous_func_code() assert os.path.exists(os.path.join(path, 'func_code.py')) assert g._check_previous_func_code() # Test the robustness to failure of loading previous results. func_id, args_id = g._get_output_identifiers(1) output_dir = os.path.join(g.store_backend.location, func_id, args_id) a = g(1) assert os.path.exists(output_dir) os.remove(os.path.join(output_dir, 'output.pkl')) assert a == g(1) def test_persistence(tmpdir): # Test the memorized functions can be pickled and restored. memory = Memory(location=tmpdir.strpath, verbose=0) g = memory.cache(f) output = g(1) h = pickle.loads(pickle.dumps(g)) func_id, args_id = h._get_output_identifiers(1) output_dir = os.path.join(h.store_backend.location, func_id, args_id) assert os.path.exists(output_dir) assert output == h.store_backend.load_item([func_id, args_id]) memory2 = pickle.loads(pickle.dumps(memory)) assert memory.store_backend.location == memory2.store_backend.location # Smoke test that pickling a memory with location=None works memory = Memory(location=None, verbose=0) pickle.loads(pickle.dumps(memory)) g = memory.cache(f) gp = pickle.loads(pickle.dumps(g)) gp(1) def test_check_call_in_cache(tmpdir): for func in (MemorizedFunc(f, tmpdir.strpath), Memory(location=tmpdir.strpath, verbose=0).cache(f)): result = func.check_call_in_cache(2) assert not result assert isinstance(result, bool) assert func(2) == 5 result = func.check_call_in_cache(2) assert result assert isinstance(result, bool) func.clear() def test_call_and_shelve(tmpdir): # Test MemorizedFunc outputting a reference to cache. for func, Result in zip((MemorizedFunc(f, tmpdir.strpath), NotMemorizedFunc(f), Memory(location=tmpdir.strpath, verbose=0).cache(f), Memory(location=None).cache(f), ), (MemorizedResult, NotMemorizedResult, MemorizedResult, NotMemorizedResult)): assert func(2) == 5 result = func.call_and_shelve(2) assert isinstance(result, Result) assert result.get() == 5 result.clear() with raises(KeyError): result.get() result.clear() # Do nothing if there is no cache. def test_call_and_shelve_argument_hash(tmpdir): # Verify that a warning is raised when accessing arguments_hash # attribute from MemorizedResult func = Memory(location=tmpdir.strpath, verbose=0).cache(f) result = func.call_and_shelve(2) assert isinstance(result, MemorizedResult) with warns(DeprecationWarning) as w: assert result.argument_hash == result.args_id assert len(w) == 1 assert "The 'argument_hash' attribute has been deprecated" \ in str(w[-1].message) def test_call_and_shelve_lazily_load_stored_result(tmpdir): """Check call_and_shelve only load stored data if needed.""" test_access_time_file = tmpdir.join('test_access') test_access_time_file.write('test_access') test_access_time = os.stat(test_access_time_file.strpath).st_atime # check file system access time stats resolution is lower than test wait # timings. time.sleep(0.5) assert test_access_time_file.read() == 'test_access' if test_access_time == os.stat(test_access_time_file.strpath).st_atime: # Skip this test when access time cannot be retrieved with enough # precision from the file system (e.g. NTFS on windows). pytest.skip("filesystem does not support fine-grained access time " "attribute") memory = Memory(location=tmpdir.strpath, verbose=0) func = memory.cache(f) func_id, argument_hash = func._get_output_identifiers(2) result_path = os.path.join(memory.store_backend.location, func_id, argument_hash, 'output.pkl') assert func(2) == 5 first_access_time = os.stat(result_path).st_atime time.sleep(1) # Should not access the stored data result = func.call_and_shelve(2) assert isinstance(result, MemorizedResult) assert os.stat(result_path).st_atime == first_access_time time.sleep(1) # Read the stored data => last access time is greater than first_access assert result.get() == 5 assert os.stat(result_path).st_atime > first_access_time def test_memorized_pickling(tmpdir): for func in (MemorizedFunc(f, tmpdir.strpath), NotMemorizedFunc(f)): filename = tmpdir.join('pickling_test.dat').strpath result = func.call_and_shelve(2) with open(filename, 'wb') as fp: pickle.dump(result, fp) with open(filename, 'rb') as fp: result2 = pickle.load(fp) assert result2.get() == result.get() os.remove(filename) def test_memorized_repr(tmpdir): func = MemorizedFunc(f, tmpdir.strpath) result = func.call_and_shelve(2) func2 = MemorizedFunc(f, tmpdir.strpath) result2 = func2.call_and_shelve(2) assert result.get() == result2.get() assert repr(func) == repr(func2) # Smoke test with NotMemorizedFunc func = NotMemorizedFunc(f) repr(func) repr(func.call_and_shelve(2)) # Smoke test for message output (increase code coverage) func = MemorizedFunc(f, tmpdir.strpath, verbose=11, timestamp=time.time()) result = func.call_and_shelve(11) result.get() func = MemorizedFunc(f, tmpdir.strpath, verbose=11) result = func.call_and_shelve(11) result.get() func = MemorizedFunc(f, tmpdir.strpath, verbose=5, timestamp=time.time()) result = func.call_and_shelve(11) result.get() func = MemorizedFunc(f, tmpdir.strpath, verbose=5) result = func.call_and_shelve(11) result.get() def test_memory_file_modification(capsys, tmpdir, monkeypatch): # Test that modifying a Python file after loading it does not lead to # Recomputation dir_name = tmpdir.mkdir('tmp_import').strpath filename = os.path.join(dir_name, 'tmp_joblib_.py') content = 'def f(x):\n print(x)\n return x\n' with open(filename, 'w') as module_file: module_file.write(content) # Load the module: monkeypatch.syspath_prepend(dir_name) import tmp_joblib_ as tmp memory = Memory(location=tmpdir.strpath, verbose=0) f = memory.cache(tmp.f) # First call f a few times f(1) f(2) f(1) # Now modify the module where f is stored without modifying f with open(filename, 'w') as module_file: module_file.write('\n\n' + content) # And call f a couple more times f(1) f(1) # Flush the .pyc files shutil.rmtree(dir_name) os.mkdir(dir_name) # Now modify the module where f is stored, modifying f content = 'def f(x):\n print("x=%s" % x)\n return x\n' with open(filename, 'w') as module_file: module_file.write(content) # And call f more times prior to reloading: the cache should not be # invalidated at this point as the active function definition has not # changed in memory yet. f(1) f(1) # Now reload sys.stdout.write('Reloading\n') sys.modules.pop('tmp_joblib_') import tmp_joblib_ as tmp f = memory.cache(tmp.f) # And call f more times f(1) f(1) out, err = capsys.readouterr() assert out == '1\n2\nReloading\nx=1\n' def _function_to_cache(a, b): # Just a place holder function to be mutated by tests pass def _sum(a, b): return a + b def _product(a, b): return a * b def test_memory_in_memory_function_code_change(tmpdir): _function_to_cache.__code__ = _sum.__code__ memory = Memory(location=tmpdir.strpath, verbose=0) f = memory.cache(_function_to_cache) assert f(1, 2) == 3 assert f(1, 2) == 3 with warns(JobLibCollisionWarning): # Check that inline function modification triggers a cache invalidation _function_to_cache.__code__ = _product.__code__ assert f(1, 2) == 2 assert f(1, 2) == 2 def test_clear_memory_with_none_location(): memory = Memory(location=None) memory.clear() def func_with_kwonly_args(a, b, *, kw1='kw1', kw2='kw2'): return a, b, kw1, kw2 def func_with_signature(a: int, b: float) -> float: return a + b def test_memory_func_with_kwonly_args(tmpdir): memory = Memory(location=tmpdir.strpath, verbose=0) func_cached = memory.cache(func_with_kwonly_args) assert func_cached(1, 2, kw1=3) == (1, 2, 3, 'kw2') # Making sure that providing a keyword-only argument by # position raises an exception with raises(ValueError) as excinfo: func_cached(1, 2, 3, kw2=4) excinfo.match("Keyword-only parameter 'kw1' was passed as positional " "parameter") # Keyword-only parameter passed by position with cached call # should still raise ValueError func_cached(1, 2, kw1=3, kw2=4) with raises(ValueError) as excinfo: func_cached(1, 2, 3, kw2=4) excinfo.match("Keyword-only parameter 'kw1' was passed as positional " "parameter") # Test 'ignore' parameter func_cached = memory.cache(func_with_kwonly_args, ignore=['kw2']) assert func_cached(1, 2, kw1=3, kw2=4) == (1, 2, 3, 4) assert func_cached(1, 2, kw1=3, kw2='ignored') == (1, 2, 3, 4) def test_memory_func_with_signature(tmpdir): memory = Memory(location=tmpdir.strpath, verbose=0) func_cached = memory.cache(func_with_signature) assert func_cached(1, 2.) == 3. def _setup_toy_cache(tmpdir, num_inputs=10): memory = Memory(location=tmpdir.strpath, verbose=0) @memory.cache() def get_1000_bytes(arg): return 'a' * 1000 inputs = list(range(num_inputs)) for arg in inputs: get_1000_bytes(arg) func_id = _build_func_identifier(get_1000_bytes) hash_dirnames = [get_1000_bytes._get_output_identifiers(arg)[1] for arg in inputs] full_hashdirs = [os.path.join(get_1000_bytes.store_backend.location, func_id, dirname) for dirname in hash_dirnames] return memory, full_hashdirs, get_1000_bytes def test__get_items(tmpdir): memory, expected_hash_dirs, _ = _setup_toy_cache(tmpdir) items = memory.store_backend.get_items() hash_dirs = [ci.path for ci in items] assert set(hash_dirs) == set(expected_hash_dirs) def get_files_size(directory): full_paths = [os.path.join(directory, fn) for fn in os.listdir(directory)] return sum(os.path.getsize(fp) for fp in full_paths) expected_hash_cache_sizes = [get_files_size(hash_dir) for hash_dir in hash_dirs] hash_cache_sizes = [ci.size for ci in items] assert hash_cache_sizes == expected_hash_cache_sizes output_filenames = [os.path.join(hash_dir, 'output.pkl') for hash_dir in hash_dirs] expected_last_accesses = [ datetime.datetime.fromtimestamp(os.path.getatime(fn)) for fn in output_filenames] last_accesses = [ci.last_access for ci in items] assert last_accesses == expected_last_accesses def test__get_items_to_delete(tmpdir): memory, expected_hash_cachedirs, _ = _setup_toy_cache(tmpdir) items = memory.store_backend.get_items() # bytes_limit set to keep only one cache item (each hash cache # folder is about 1000 bytes + metadata) items_to_delete = memory.store_backend._get_items_to_delete('2K') nb_hashes = len(expected_hash_cachedirs) assert set.issubset(set(items_to_delete), set(items)) assert len(items_to_delete) == nb_hashes - 1 # Sanity check bytes_limit=2048 is the same as bytes_limit='2K' items_to_delete_2048b = memory.store_backend._get_items_to_delete(2048) assert sorted(items_to_delete) == sorted(items_to_delete_2048b) # bytes_limit greater than the size of the cache items_to_delete_empty = memory.store_backend._get_items_to_delete('1M') assert items_to_delete_empty == [] # All the cache items need to be deleted bytes_limit_too_small = 500 items_to_delete_500b = memory.store_backend._get_items_to_delete( bytes_limit_too_small ) assert set(items_to_delete_500b), set(items) # Test LRU property: surviving cache items should all have a more # recent last_access that the ones that have been deleted items_to_delete_6000b = memory.store_backend._get_items_to_delete(6000) surviving_items = set(items).difference(items_to_delete_6000b) assert (max(ci.last_access for ci in items_to_delete_6000b) <= min(ci.last_access for ci in surviving_items)) def test_memory_reduce_size_bytes_limit(tmpdir): memory, _, _ = _setup_toy_cache(tmpdir) ref_cache_items = memory.store_backend.get_items() # By default memory.bytes_limit is None and reduce_size is a noop memory.reduce_size() cache_items = memory.store_backend.get_items() assert sorted(ref_cache_items) == sorted(cache_items) # No cache items deleted if bytes_limit greater than the size of # the cache memory.reduce_size(bytes_limit='1M') cache_items = memory.store_backend.get_items() assert sorted(ref_cache_items) == sorted(cache_items) # bytes_limit is set so that only two cache items are kept memory.reduce_size(bytes_limit='3K') cache_items = memory.store_backend.get_items() assert set.issubset(set(cache_items), set(ref_cache_items)) assert len(cache_items) == 2 # bytes_limit set so that no cache item is kept bytes_limit_too_small = 500 memory.reduce_size(bytes_limit=bytes_limit_too_small) cache_items = memory.store_backend.get_items() assert cache_items == [] def test_memory_reduce_size_items_limit(tmpdir): memory, _, _ = _setup_toy_cache(tmpdir) ref_cache_items = memory.store_backend.get_items() # By default reduce_size is a noop memory.reduce_size() cache_items = memory.store_backend.get_items() assert sorted(ref_cache_items) == sorted(cache_items) # No cache items deleted if items_limit greater than the size of # the cache memory.reduce_size(items_limit=10) cache_items = memory.store_backend.get_items() assert sorted(ref_cache_items) == sorted(cache_items) # items_limit is set so that only two cache items are kept memory.reduce_size(items_limit=2) cache_items = memory.store_backend.get_items() assert set.issubset(set(cache_items), set(ref_cache_items)) assert len(cache_items) == 2 # item_limit set so that no cache item is kept memory.reduce_size(items_limit=0) cache_items = memory.store_backend.get_items() assert cache_items == [] def test_memory_reduce_size_age_limit(tmpdir): import time import datetime memory, _, put_cache = _setup_toy_cache(tmpdir) ref_cache_items = memory.store_backend.get_items() # By default reduce_size is a noop memory.reduce_size() cache_items = memory.store_backend.get_items() assert sorted(ref_cache_items) == sorted(cache_items) # No cache items deleted if age_limit big. memory.reduce_size(age_limit=datetime.timedelta(days=1)) cache_items = memory.store_backend.get_items() assert sorted(ref_cache_items) == sorted(cache_items) # age_limit is set so that only two cache items are kept time.sleep(1) put_cache(-1) put_cache(-2) memory.reduce_size(age_limit=datetime.timedelta(seconds=1)) cache_items = memory.store_backend.get_items() assert not set.issubset(set(cache_items), set(ref_cache_items)) assert len(cache_items) == 2 # age_limit set so that no cache item is kept memory.reduce_size(age_limit=datetime.timedelta(seconds=0)) cache_items = memory.store_backend.get_items() assert cache_items == [] def test_memory_clear(tmpdir): memory, _, g = _setup_toy_cache(tmpdir) memory.clear() assert os.listdir(memory.store_backend.location) == [] # Check that the cache for functions hash is also reset. assert not g._check_previous_func_code(stacklevel=4) def fast_func_with_complex_output(): complex_obj = ['a' * 1000] * 1000 return complex_obj def fast_func_with_conditional_complex_output(complex_output=True): complex_obj = {str(i): i for i in range(int(1e5))} return complex_obj if complex_output else 'simple output' @with_multiprocessing def test_cached_function_race_condition_when_persisting_output(tmpdir, capfd): # Test race condition where multiple processes are writing into # the same output.pkl. See # https://github.com/joblib/joblib/issues/490 for more details. memory = Memory(location=tmpdir.strpath) func_cached = memory.cache(fast_func_with_complex_output) Parallel(n_jobs=2)(delayed(func_cached)() for i in range(3)) stdout, stderr = capfd.readouterr() # Checking both stdout and stderr (ongoing PR #434 may change # logging destination) to make sure there is no exception while # loading the results exception_msg = 'Exception while loading results' assert exception_msg not in stdout assert exception_msg not in stderr @with_multiprocessing def test_cached_function_race_condition_when_persisting_output_2(tmpdir, capfd): # Test race condition in first attempt at solving # https://github.com/joblib/joblib/issues/490. The race condition # was due to the delay between seeing the cache directory created # (interpreted as the result being cached) and the output.pkl being # pickled. memory = Memory(location=tmpdir.strpath) func_cached = memory.cache(fast_func_with_conditional_complex_output) Parallel(n_jobs=2)(delayed(func_cached)(True if i % 2 == 0 else False) for i in range(3)) stdout, stderr = capfd.readouterr() # Checking both stdout and stderr (ongoing PR #434 may change # logging destination) to make sure there is no exception while # loading the results exception_msg = 'Exception while loading results' assert exception_msg not in stdout assert exception_msg not in stderr def test_memory_recomputes_after_an_error_while_loading_results( tmpdir, monkeypatch): memory = Memory(location=tmpdir.strpath) def func(arg): # This makes sure that the timestamp returned by two calls of # func are different. This is needed on Windows where # time.time resolution may not be accurate enough time.sleep(0.01) return arg, time.time() cached_func = memory.cache(func) input_arg = 'arg' arg, timestamp = cached_func(input_arg) # Make sure the function is correctly cached assert arg == input_arg # Corrupting output.pkl to make sure that an error happens when # loading the cached result corrupt_single_cache_item(memory) # Make sure that corrupting the file causes recomputation and that # a warning is issued. recorded_warnings = monkeypatch_cached_func_warn(cached_func, monkeypatch) recomputed_arg, recomputed_timestamp = cached_func(arg) assert len(recorded_warnings) == 1 exception_msg = 'Exception while loading results' assert exception_msg in recorded_warnings[0] assert recomputed_arg == arg assert recomputed_timestamp > timestamp # Corrupting output.pkl to make sure that an error happens when # loading the cached result corrupt_single_cache_item(memory) reference = cached_func.call_and_shelve(arg) try: reference.get() raise AssertionError( "It normally not possible to load a corrupted" " MemorizedResult" ) except KeyError as e: message = "is corrupted" assert message in str(e.args) class IncompleteStoreBackend(StoreBackendBase): """This backend cannot be instantiated and should raise a TypeError.""" pass class DummyStoreBackend(StoreBackendBase): """A dummy store backend that does nothing.""" def _open_item(self, *args, **kwargs): """Open an item on store.""" "Does nothing" def _item_exists(self, location): """Check if an item location exists.""" "Does nothing" def _move_item(self, src, dst): """Move an item from src to dst in store.""" "Does nothing" def create_location(self, location): """Create location on store.""" "Does nothing" def exists(self, obj): """Check if an object exists in the store""" return False def clear_location(self, obj): """Clear object on store""" "Does nothing" def get_items(self): """Returns the whole list of items available in cache.""" return [] def configure(self, location, *args, **kwargs): """Configure the store""" "Does nothing" @parametrize("invalid_prefix", [None, dict(), list()]) def test_register_invalid_store_backends_key(invalid_prefix): # verify the right exceptions are raised when passing a wrong backend key. with raises(ValueError) as excinfo: register_store_backend(invalid_prefix, None) excinfo.match(r'Store backend name should be a string*') def test_register_invalid_store_backends_object(): # verify the right exceptions are raised when passing a wrong backend # object. with raises(ValueError) as excinfo: register_store_backend("fs", None) excinfo.match(r'Store backend should inherit StoreBackendBase*') def test_memory_default_store_backend(): # test an unknown backend falls back into a FileSystemStoreBackend with raises(TypeError) as excinfo: Memory(location='/tmp/joblib', backend='unknown') excinfo.match(r"Unknown location*") def test_warning_on_unknown_location_type(): class NonSupportedLocationClass: pass unsupported_location = NonSupportedLocationClass() with warns(UserWarning) as warninfo: _store_backend_factory("local", location=unsupported_location) expected_mesage = ("Instantiating a backend using a " "NonSupportedLocationClass as a location is not " "supported by joblib") assert expected_mesage in str(warninfo[0].message) def test_instanciate_incomplete_store_backend(): # Verify that registering an external incomplete store backend raises an # exception when one tries to instantiate it. backend_name = "isb" register_store_backend(backend_name, IncompleteStoreBackend) assert (backend_name, IncompleteStoreBackend) in _STORE_BACKENDS.items() with raises(TypeError) as excinfo: _store_backend_factory(backend_name, "fake_location") excinfo.match(r"Can't instantiate abstract class IncompleteStoreBackend " "(without an implementation for|with) abstract methods*") def test_dummy_store_backend(): # Verify that registering an external store backend works. backend_name = "dsb" register_store_backend(backend_name, DummyStoreBackend) assert (backend_name, DummyStoreBackend) in _STORE_BACKENDS.items() backend_obj = _store_backend_factory(backend_name, "dummy_location") assert isinstance(backend_obj, DummyStoreBackend) def test_instanciate_store_backend_with_pathlib_path(): # Instantiate a FileSystemStoreBackend using a pathlib.Path object path = pathlib.Path("some_folder") backend_obj = _store_backend_factory("local", path) assert backend_obj.location == "some_folder" def test_filesystem_store_backend_repr(tmpdir): # Verify string representation of a filesystem store backend. repr_pattern = 'FileSystemStoreBackend(location="{location}")' backend = FileSystemStoreBackend() assert backend.location is None repr(backend) # Should not raise an exception assert str(backend) == repr_pattern.format(location=None) # backend location is passed explicitly via the configure method (called # by the internal _store_backend_factory function) backend.configure(tmpdir.strpath) assert str(backend) == repr_pattern.format(location=tmpdir.strpath) repr(backend) # Should not raise an exception def test_memory_objects_repr(tmpdir): # Verify printable reprs of MemorizedResult, MemorizedFunc and Memory. def my_func(a, b): return a + b memory = Memory(location=tmpdir.strpath, verbose=0) memorized_func = memory.cache(my_func) memorized_func_repr = 'MemorizedFunc(func={func}, location={location})' assert str(memorized_func) == memorized_func_repr.format( func=my_func, location=memory.store_backend.location) memorized_result = memorized_func.call_and_shelve(42, 42) memorized_result_repr = ('MemorizedResult(location="{location}", ' 'func="{func}", args_id="{args_id}")') assert str(memorized_result) == memorized_result_repr.format( location=memory.store_backend.location, func=memorized_result.func_id, args_id=memorized_result.args_id) assert str(memory) == 'Memory(location={location})'.format( location=memory.store_backend.location) def test_memorized_result_pickle(tmpdir): # Verify a MemoryResult object can be pickled/depickled. Non regression # test introduced following issue # https://github.com/joblib/joblib/issues/747 memory = Memory(location=tmpdir.strpath) @memory.cache def g(x): return x**2 memorized_result = g.call_and_shelve(4) memorized_result_pickle = pickle.dumps(memorized_result) memorized_result_loads = pickle.loads(memorized_result_pickle) assert memorized_result.store_backend.location == \ memorized_result_loads.store_backend.location assert memorized_result.func == memorized_result_loads.func assert memorized_result.args_id == memorized_result_loads.args_id assert str(memorized_result) == str(memorized_result_loads) def compare(left, right, ignored_attrs=None): if ignored_attrs is None: ignored_attrs = [] left_vars = vars(left) right_vars = vars(right) assert set(left_vars.keys()) == set(right_vars.keys()) for attr in left_vars.keys(): if attr in ignored_attrs: continue assert left_vars[attr] == right_vars[attr] @pytest.mark.parametrize('memory_kwargs', [{'compress': 3, 'verbose': 2}, {'mmap_mode': 'r', 'verbose': 5, 'backend_options': {'parameter': 'unused'}}]) def test_memory_pickle_dump_load(tmpdir, memory_kwargs): memory = Memory(location=tmpdir.strpath, **memory_kwargs) memory_reloaded = pickle.loads(pickle.dumps(memory)) # Compare Memory instance before and after pickle roundtrip compare(memory.store_backend, memory_reloaded.store_backend) compare(memory, memory_reloaded, ignored_attrs=set(['store_backend', 'timestamp', '_func_code_id'])) assert hash(memory) == hash(memory_reloaded) func_cached = memory.cache(f) func_cached_reloaded = pickle.loads(pickle.dumps(func_cached)) # Compare MemorizedFunc instance before/after pickle roundtrip compare(func_cached.store_backend, func_cached_reloaded.store_backend) compare(func_cached, func_cached_reloaded, ignored_attrs=set(['store_backend', 'timestamp', '_func_code_id'])) assert hash(func_cached) == hash(func_cached_reloaded) # Compare MemorizedResult instance before/after pickle roundtrip memorized_result = func_cached.call_and_shelve(1) memorized_result_reloaded = pickle.loads(pickle.dumps(memorized_result)) compare(memorized_result.store_backend, memorized_result_reloaded.store_backend) compare(memorized_result, memorized_result_reloaded, ignored_attrs=set(['store_backend', 'timestamp', '_func_code_id'])) assert hash(memorized_result) == hash(memorized_result_reloaded) def test_info_log(tmpdir, caplog): caplog.set_level(logging.INFO) x = 3 memory = Memory(location=tmpdir.strpath, verbose=20) @memory.cache def f(x): return x ** 2 _ = f(x) assert "Querying" in caplog.text caplog.clear() memory = Memory(location=tmpdir.strpath, verbose=0) @memory.cache def f(x): return x ** 2 _ = f(x) assert "Querying" not in caplog.text caplog.clear() def test_deprecated_bytes_limit(tmpdir): from joblib import __version__ if __version__ >= "1.5": raise DeprecationWarning( "Bytes limit is deprecated and should be removed by 1.4" ) with pytest.warns(DeprecationWarning, match="bytes_limit"): _ = Memory(location=tmpdir.strpath, bytes_limit='1K') class TestCacheValidationCallback: "Tests on parameter `cache_validation_callback`" @pytest.fixture() def memory(self, tmp_path): mem = Memory(location=tmp_path) yield mem mem.clear() def foo(self, x, d, delay=None): d["run"] = True if delay is not None: time.sleep(delay) return x * 2 def test_invalid_cache_validation_callback(self, memory): "Test invalid values for `cache_validation_callback" match = "cache_validation_callback needs to be callable. Got True." with pytest.raises(ValueError, match=match): memory.cache(cache_validation_callback=True) @pytest.mark.parametrize("consider_cache_valid", [True, False]) def test_constant_cache_validation_callback( self, memory, consider_cache_valid ): "Test expiry of old results" f = memory.cache( self.foo, cache_validation_callback=lambda _: consider_cache_valid, ignore=["d"] ) d1, d2 = {"run": False}, {"run": False} assert f(2, d1) == 4 assert f(2, d2) == 4 assert d1["run"] assert d2["run"] != consider_cache_valid def test_memory_only_cache_long_run(self, memory): "Test cache validity based on run duration." def cache_validation_callback(metadata): duration = metadata['duration'] if duration > 0.1: return True f = memory.cache( self.foo, cache_validation_callback=cache_validation_callback, ignore=["d"] ) # Short run are not cached d1, d2 = {"run": False}, {"run": False} assert f(2, d1, delay=0) == 4 assert f(2, d2, delay=0) == 4 assert d1["run"] assert d2["run"] # Longer run are cached d1, d2 = {"run": False}, {"run": False} assert f(2, d1, delay=0.2) == 4 assert f(2, d2, delay=0.2) == 4 assert d1["run"] assert not d2["run"] def test_memory_expires_after(self, memory): "Test expiry of old cached results" f = memory.cache( self.foo, cache_validation_callback=expires_after(seconds=.3), ignore=["d"] ) d1, d2, d3 = {"run": False}, {"run": False}, {"run": False} assert f(2, d1) == 4 assert f(2, d2) == 4 time.sleep(.5) assert f(2, d3) == 4 assert d1["run"] assert not d2["run"] assert d3["run"] joblib-1.3.2/joblib/test/test_missing_multiprocessing.py000066400000000000000000000021431446465525000236020ustar00rootroot00000000000000""" Pyodide and other single-threaded Python builds will be missing the _multiprocessing module. Test that joblib still works in this environment. """ import os import subprocess import sys def test_missing_multiprocessing(tmp_path): """ Test that import joblib works even if _multiprocessing is missing. pytest has already imported everything from joblib. The most reasonable way to test importing joblib with modified environment is to invoke a separate Python process. This also ensures that we don't break other tests by importing a bad `_multiprocessing` module. """ (tmp_path / "_multiprocessing.py").write_text( 'raise ImportError("No _multiprocessing module!")' ) env = dict(os.environ) # For subprocess, use current sys.path with our custom version of # multiprocessing inserted. env["PYTHONPATH"] = ":".join([str(tmp_path)] + sys.path) subprocess.check_call( [sys.executable, "-c", "import joblib, math; " "joblib.Parallel(n_jobs=1)(" "joblib.delayed(math.sqrt)(i**2) for i in range(10))" ], env=env) joblib-1.3.2/joblib/test/test_module.py000066400000000000000000000036201446465525000201100ustar00rootroot00000000000000import sys import joblib from joblib.testing import check_subprocess_call from joblib.test.common import with_multiprocessing def test_version(): assert hasattr(joblib, '__version__'), ( "There are no __version__ argument on the joblib module") @with_multiprocessing def test_no_start_method_side_effect_on_import(): # check that importing joblib does not implicitly set the global # start_method for multiprocessing. code = """if True: import joblib import multiprocessing as mp # The following line would raise RuntimeError if the # start_method is already set. mp.set_start_method("loky") """ check_subprocess_call([sys.executable, '-c', code]) @with_multiprocessing def test_no_semaphore_tracker_on_import(): # check that importing joblib does not implicitly spawn a resource tracker # or a semaphore tracker code = """if True: import joblib from multiprocessing import semaphore_tracker # The following line would raise RuntimeError if the # start_method is already set. msg = "multiprocessing.semaphore_tracker has been spawned on import" assert semaphore_tracker._semaphore_tracker._fd is None, msg""" if sys.version_info >= (3, 8): # semaphore_tracker was renamed in Python 3.8: code = code.replace("semaphore_tracker", "resource_tracker") check_subprocess_call([sys.executable, '-c', code]) @with_multiprocessing def test_no_resource_tracker_on_import(): code = """if True: import joblib from joblib.externals.loky.backend import resource_tracker # The following line would raise RuntimeError if the # start_method is already set. msg = "loky.resource_tracker has been spawned on import" assert resource_tracker._resource_tracker._fd is None, msg """ check_subprocess_call([sys.executable, '-c', code]) joblib-1.3.2/joblib/test/test_numpy_pickle.py000066400000000000000000001227651446465525000213360ustar00rootroot00000000000000"""Test the numpy pickler as a replacement of the standard pickler.""" import copy import os import random import re import io import sys import warnings import gzip import zlib import bz2 import pickle import socket from contextlib import closing import mmap from pathlib import Path try: import lzma except ImportError: lzma = None import pytest from joblib.test.common import np, with_numpy, with_lz4, without_lz4 from joblib.test.common import with_memory_profiler, memory_used from joblib.testing import parametrize, raises, warns # numpy_pickle is not a drop-in replacement of pickle, as it takes # filenames instead of open files as arguments. from joblib import numpy_pickle, register_compressor from joblib.test import data from joblib.numpy_pickle_utils import _IO_BUFFER_SIZE from joblib.numpy_pickle_utils import _detect_compressor from joblib.numpy_pickle_utils import _is_numpy_array_byte_order_mismatch from joblib.numpy_pickle_utils import _ensure_native_byte_order from joblib.compressor import (_COMPRESSORS, _LZ4_PREFIX, CompressorWrapper, LZ4_NOT_INSTALLED_ERROR, BinaryZlibFile) ############################################################################### # Define a list of standard types. # Borrowed from dill, initial author: Micheal McKerns: # http://dev.danse.us/trac/pathos/browser/dill/dill_test2.py typelist = [] # testing types _none = None typelist.append(_none) _type = type typelist.append(_type) _bool = bool(1) typelist.append(_bool) _int = int(1) typelist.append(_int) _float = float(1) typelist.append(_float) _complex = complex(1) typelist.append(_complex) _string = str(1) typelist.append(_string) _tuple = () typelist.append(_tuple) _list = [] typelist.append(_list) _dict = {} typelist.append(_dict) _builtin = len typelist.append(_builtin) def _function(x): yield x class _class: def _method(self): pass class _newclass(object): def _method(self): pass typelist.append(_function) typelist.append(_class) typelist.append(_newclass) # _instance = _class() typelist.append(_instance) _object = _newclass() typelist.append(_object) # ############################################################################### # Tests @parametrize('compress', [0, 1]) @parametrize('member', typelist) def test_standard_types(tmpdir, compress, member): # Test pickling and saving with standard types. filename = tmpdir.join('test.pkl').strpath numpy_pickle.dump(member, filename, compress=compress) _member = numpy_pickle.load(filename) # We compare the pickled instance to the reloaded one only if it # can be compared to a copied one if member == copy.deepcopy(member): assert member == _member def test_value_error(): # Test inverting the input arguments to dump with raises(ValueError): numpy_pickle.dump('foo', dict()) @parametrize('wrong_compress', [-1, 10, dict()]) def test_compress_level_error(wrong_compress): # Verify that passing an invalid compress argument raises an error. exception_msg = ('Non valid compress level given: ' '"{0}"'.format(wrong_compress)) with raises(ValueError) as excinfo: numpy_pickle.dump('dummy', 'foo', compress=wrong_compress) excinfo.match(exception_msg) @with_numpy @parametrize('compress', [False, True, 0, 3, 'zlib']) def test_numpy_persistence(tmpdir, compress): filename = tmpdir.join('test.pkl').strpath rnd = np.random.RandomState(0) a = rnd.random_sample((10, 2)) # We use 'a.T' to have a non C-contiguous array. for index, obj in enumerate(((a,), (a.T,), (a, a), [a, a, a])): filenames = numpy_pickle.dump(obj, filename, compress=compress) # All is cached in one file assert len(filenames) == 1 # Check that only one file was created assert filenames[0] == filename # Check that this file does exist assert os.path.exists(filenames[0]) # Unpickle the object obj_ = numpy_pickle.load(filename) # Check that the items are indeed arrays for item in obj_: assert isinstance(item, np.ndarray) # And finally, check that all the values are equal. np.testing.assert_array_equal(np.array(obj), np.array(obj_)) # Now test with an array subclass obj = np.memmap(filename + 'mmap', mode='w+', shape=4, dtype=np.float64) filenames = numpy_pickle.dump(obj, filename, compress=compress) # All is cached in one file assert len(filenames) == 1 obj_ = numpy_pickle.load(filename) if (type(obj) is not np.memmap and hasattr(obj, '__array_prepare__')): # We don't reconstruct memmaps assert isinstance(obj_, type(obj)) np.testing.assert_array_equal(obj_, obj) # Test with an object containing multiple numpy arrays obj = ComplexTestObject() filenames = numpy_pickle.dump(obj, filename, compress=compress) # All is cached in one file assert len(filenames) == 1 obj_loaded = numpy_pickle.load(filename) assert isinstance(obj_loaded, type(obj)) np.testing.assert_array_equal(obj_loaded.array_float, obj.array_float) np.testing.assert_array_equal(obj_loaded.array_int, obj.array_int) np.testing.assert_array_equal(obj_loaded.array_obj, obj.array_obj) @with_numpy def test_numpy_persistence_bufferred_array_compression(tmpdir): big_array = np.ones((_IO_BUFFER_SIZE + 100), dtype=np.uint8) filename = tmpdir.join('test.pkl').strpath numpy_pickle.dump(big_array, filename, compress=True) arr_reloaded = numpy_pickle.load(filename) np.testing.assert_array_equal(big_array, arr_reloaded) @with_numpy def test_memmap_persistence(tmpdir): rnd = np.random.RandomState(0) a = rnd.random_sample(10) filename = tmpdir.join('test1.pkl').strpath numpy_pickle.dump(a, filename) b = numpy_pickle.load(filename, mmap_mode='r') assert isinstance(b, np.memmap) # Test with an object containing multiple numpy arrays filename = tmpdir.join('test2.pkl').strpath obj = ComplexTestObject() numpy_pickle.dump(obj, filename) obj_loaded = numpy_pickle.load(filename, mmap_mode='r') assert isinstance(obj_loaded, type(obj)) assert isinstance(obj_loaded.array_float, np.memmap) assert not obj_loaded.array_float.flags.writeable assert isinstance(obj_loaded.array_int, np.memmap) assert not obj_loaded.array_int.flags.writeable # Memory map not allowed for numpy object arrays assert not isinstance(obj_loaded.array_obj, np.memmap) np.testing.assert_array_equal(obj_loaded.array_float, obj.array_float) np.testing.assert_array_equal(obj_loaded.array_int, obj.array_int) np.testing.assert_array_equal(obj_loaded.array_obj, obj.array_obj) # Test we can write in memmapped arrays obj_loaded = numpy_pickle.load(filename, mmap_mode='r+') assert obj_loaded.array_float.flags.writeable obj_loaded.array_float[0:10] = 10.0 assert obj_loaded.array_int.flags.writeable obj_loaded.array_int[0:10] = 10 obj_reloaded = numpy_pickle.load(filename, mmap_mode='r') np.testing.assert_array_equal(obj_reloaded.array_float, obj_loaded.array_float) np.testing.assert_array_equal(obj_reloaded.array_int, obj_loaded.array_int) # Test w+ mode is caught and the mode has switched to r+ numpy_pickle.load(filename, mmap_mode='w+') assert obj_loaded.array_int.flags.writeable assert obj_loaded.array_int.mode == 'r+' assert obj_loaded.array_float.flags.writeable assert obj_loaded.array_float.mode == 'r+' @with_numpy def test_memmap_persistence_mixed_dtypes(tmpdir): # loading datastructures that have sub-arrays with dtype=object # should not prevent memmapping on fixed size dtype sub-arrays. rnd = np.random.RandomState(0) a = rnd.random_sample(10) b = np.array([1, 'b'], dtype=object) construct = (a, b) filename = tmpdir.join('test.pkl').strpath numpy_pickle.dump(construct, filename) a_clone, b_clone = numpy_pickle.load(filename, mmap_mode='r') # the floating point array has been memory mapped assert isinstance(a_clone, np.memmap) # the object-dtype array has been loaded in memory assert not isinstance(b_clone, np.memmap) @with_numpy def test_masked_array_persistence(tmpdir): # The special-case picker fails, because saving masked_array # not implemented, but it just delegates to the standard pickler. rnd = np.random.RandomState(0) a = rnd.random_sample(10) a = np.ma.masked_greater(a, 0.5) filename = tmpdir.join('test.pkl').strpath numpy_pickle.dump(a, filename) b = numpy_pickle.load(filename, mmap_mode='r') assert isinstance(b, np.ma.masked_array) @with_numpy def test_compress_mmap_mode_warning(tmpdir): # Test the warning in case of compress + mmap_mode rnd = np.random.RandomState(0) a = rnd.random_sample(10) this_filename = tmpdir.join('test.pkl').strpath numpy_pickle.dump(a, this_filename, compress=1) with warns(UserWarning) as warninfo: numpy_pickle.load(this_filename, mmap_mode='r+') debug_msg = "\n".join([str(w) for w in warninfo]) warninfo = [w.message for w in warninfo] assert len(warninfo) == 1, debug_msg assert ( str(warninfo[0]) == 'mmap_mode "r+" is not compatible with compressed ' f'file {this_filename}. "r+" flag will be ignored.' ) @with_numpy @parametrize('cache_size', [None, 0, 10]) def test_cache_size_warning(tmpdir, cache_size): # Check deprecation warning raised when cache size is not None filename = tmpdir.join('test.pkl').strpath rnd = np.random.RandomState(0) a = rnd.random_sample((10, 2)) warnings.simplefilter("always") with warnings.catch_warnings(record=True) as warninfo: numpy_pickle.dump(a, filename, cache_size=cache_size) expected_nb_warnings = 1 if cache_size is not None else 0 assert len(warninfo) == expected_nb_warnings for w in warninfo: assert w.category == DeprecationWarning assert (str(w.message) == "Please do not set 'cache_size' in joblib.dump, this " "parameter has no effect and will be removed. You " "used 'cache_size={0}'".format(cache_size)) @with_numpy @with_memory_profiler @parametrize('compress', [True, False]) def test_memory_usage(tmpdir, compress): # Verify memory stays within expected bounds. filename = tmpdir.join('test.pkl').strpath small_array = np.ones((10, 10)) big_array = np.ones(shape=100 * int(1e6), dtype=np.uint8) for obj in (small_array, big_array): size = obj.nbytes / 1e6 obj_filename = filename + str(np.random.randint(0, 1000)) mem_used = memory_used(numpy_pickle.dump, obj, obj_filename, compress=compress) # The memory used to dump the object shouldn't exceed the buffer # size used to write array chunks (16MB). write_buf_size = _IO_BUFFER_SIZE + 16 * 1024 ** 2 / 1e6 assert mem_used <= write_buf_size mem_used = memory_used(numpy_pickle.load, obj_filename) # memory used should be less than array size + buffer size used to # read the array chunk by chunk. read_buf_size = 32 + _IO_BUFFER_SIZE # MiB assert mem_used < size + read_buf_size @with_numpy def test_compressed_pickle_dump_and_load(tmpdir): expected_list = [np.arange(5, dtype=np.dtype('i8')), np.arange(5, dtype=np.dtype('f8')), np.array([1, 'abc', {'a': 1, 'b': 2}], dtype='O'), np.arange(256, dtype=np.uint8).tobytes(), u"C'est l'\xe9t\xe9 !"] fname = tmpdir.join('temp.pkl.gz').strpath dumped_filenames = numpy_pickle.dump(expected_list, fname, compress=1) assert len(dumped_filenames) == 1 result_list = numpy_pickle.load(fname) for result, expected in zip(result_list, expected_list): if isinstance(expected, np.ndarray): expected = _ensure_native_byte_order(expected) assert result.dtype == expected.dtype np.testing.assert_equal(result, expected) else: assert result == expected def _check_pickle(filename, expected_list, mmap_mode=None): """Helper function to test joblib pickle content. Note: currently only pickles containing an iterable are supported by this function. """ version_match = re.match(r'.+py(\d)(\d).+', filename) py_version_used_for_writing = int(version_match.group(1)) py_version_to_default_pickle_protocol = {2: 2, 3: 3} pickle_reading_protocol = py_version_to_default_pickle_protocol.get(3, 4) pickle_writing_protocol = py_version_to_default_pickle_protocol.get( py_version_used_for_writing, 4) if pickle_reading_protocol >= pickle_writing_protocol: try: with warnings.catch_warnings(record=True) as warninfo: warnings.simplefilter('always') warnings.filterwarnings( 'ignore', module='numpy', message='The compiler package is deprecated') result_list = numpy_pickle.load(filename, mmap_mode=mmap_mode) filename_base = os.path.basename(filename) expected_nb_deprecation_warnings = 1 if ( "_0.9" in filename_base or "_0.8.4" in filename_base) else 0 expected_nb_user_warnings = 3 if ( re.search("_0.1.+.pkl$", filename_base) and mmap_mode is not None) else 0 expected_nb_warnings = \ expected_nb_deprecation_warnings + expected_nb_user_warnings assert len(warninfo) == expected_nb_warnings deprecation_warnings = [ w for w in warninfo if issubclass( w.category, DeprecationWarning)] user_warnings = [ w for w in warninfo if issubclass( w.category, UserWarning)] for w in deprecation_warnings: assert (str(w.message) == "The file '{0}' has been generated with a joblib " "version less than 0.10. Please regenerate this " "pickle file.".format(filename)) for w in user_warnings: escaped_filename = re.escape(filename) assert re.search( f"memmapped.+{escaped_filename}.+segmentation fault", str(w.message)) for result, expected in zip(result_list, expected_list): if isinstance(expected, np.ndarray): expected = _ensure_native_byte_order(expected) assert result.dtype == expected.dtype np.testing.assert_equal(result, expected) else: assert result == expected except Exception as exc: # When trying to read with python 3 a pickle generated # with python 2 we expect a user-friendly error if py_version_used_for_writing == 2: assert isinstance(exc, ValueError) message = ('You may be trying to read with ' 'python 3 a joblib pickle generated with python 2.') assert message in str(exc) elif filename.endswith('.lz4') and with_lz4.args[0]: assert isinstance(exc, ValueError) assert LZ4_NOT_INSTALLED_ERROR in str(exc) else: raise else: # Pickle protocol used for writing is too high. We expect a # "unsupported pickle protocol" error message try: numpy_pickle.load(filename) raise AssertionError('Numpy pickle loading should ' 'have raised a ValueError exception') except ValueError as e: message = 'unsupported pickle protocol: {0}'.format( pickle_writing_protocol) assert message in str(e.args) @with_numpy def test_joblib_pickle_across_python_versions(): # We need to be specific about dtypes in particular endianness # because the pickles can be generated on one architecture and # the tests run on another one. See # https://github.com/joblib/joblib/issues/279. expected_list = [np.arange(5, dtype=np.dtype('i8'), ('', '>f8')]), np.arange(3, dtype=np.dtype('>i8')), np.arange(3, dtype=np.dtype('>f8'))] # Verify the byteorder mismatch is correctly detected. for array in be_arrays: if sys.byteorder == 'big': assert not _is_numpy_array_byte_order_mismatch(array) else: assert _is_numpy_array_byte_order_mismatch(array) converted = _ensure_native_byte_order(array) if converted.dtype.fields: for f in converted.dtype.fields.values(): f[0].byteorder == '=' else: assert converted.dtype.byteorder == "=" # List of numpy arrays with little endian byteorder. le_arrays = [np.array([(1, 2.0), (3, 4.0)], dtype=[('', ' size np.testing.assert_array_equal(obj, memmaps) def test_register_compressor(tmpdir): # Check that registering compressor file works. compressor_name = 'test-name' compressor_prefix = 'test-prefix' class BinaryCompressorTestFile(io.BufferedIOBase): pass class BinaryCompressorTestWrapper(CompressorWrapper): def __init__(self): CompressorWrapper.__init__(self, obj=BinaryCompressorTestFile, prefix=compressor_prefix) register_compressor(compressor_name, BinaryCompressorTestWrapper()) assert (_COMPRESSORS[compressor_name].fileobj_factory == BinaryCompressorTestFile) assert _COMPRESSORS[compressor_name].prefix == compressor_prefix # Remove this dummy compressor file from extra compressors because other # tests might fail because of this _COMPRESSORS.pop(compressor_name) @parametrize('invalid_name', [1, (), {}]) def test_register_compressor_invalid_name(invalid_name): # Test that registering an invalid compressor name is not allowed. with raises(ValueError) as excinfo: register_compressor(invalid_name, None) excinfo.match("Compressor name should be a string") def test_register_compressor_invalid_fileobj(): # Test that registering an invalid file object is not allowed. class InvalidFileObject(): pass class InvalidFileObjectWrapper(CompressorWrapper): def __init__(self): CompressorWrapper.__init__(self, obj=InvalidFileObject, prefix=b'prefix') with raises(ValueError) as excinfo: register_compressor('invalid', InvalidFileObjectWrapper()) excinfo.match("Compressor 'fileobj_factory' attribute should implement " "the file object interface") class AnotherZlibCompressorWrapper(CompressorWrapper): def __init__(self): CompressorWrapper.__init__(self, obj=BinaryZlibFile, prefix=b'prefix') class StandardLibGzipCompressorWrapper(CompressorWrapper): def __init__(self): CompressorWrapper.__init__(self, obj=gzip.GzipFile, prefix=b'prefix') def test_register_compressor_already_registered(): # Test registration of existing compressor files. compressor_name = 'test-name' # register a test compressor register_compressor(compressor_name, AnotherZlibCompressorWrapper()) with raises(ValueError) as excinfo: register_compressor(compressor_name, StandardLibGzipCompressorWrapper()) excinfo.match("Compressor '{}' already registered." .format(compressor_name)) register_compressor(compressor_name, StandardLibGzipCompressorWrapper(), force=True) assert compressor_name in _COMPRESSORS assert _COMPRESSORS[compressor_name].fileobj_factory == gzip.GzipFile # Remove this dummy compressor file from extra compressors because other # tests might fail because of this _COMPRESSORS.pop(compressor_name) @with_lz4 def test_lz4_compression(tmpdir): # Check that lz4 can be used when dependency is available. import lz4.frame compressor = 'lz4' assert compressor in _COMPRESSORS assert _COMPRESSORS[compressor].fileobj_factory == lz4.frame.LZ4FrameFile fname = tmpdir.join('test.pkl').strpath data = 'test data' numpy_pickle.dump(data, fname, compress=compressor) with open(fname, 'rb') as f: assert f.read(len(_LZ4_PREFIX)) == _LZ4_PREFIX assert numpy_pickle.load(fname) == data # Test that LZ4 is applied based on file extension numpy_pickle.dump(data, fname + '.lz4') with open(fname, 'rb') as f: assert f.read(len(_LZ4_PREFIX)) == _LZ4_PREFIX assert numpy_pickle.load(fname) == data @without_lz4 def test_lz4_compression_without_lz4(tmpdir): # Check that lz4 cannot be used when dependency is not available. fname = tmpdir.join('test.nolz4').strpath data = 'test data' msg = LZ4_NOT_INSTALLED_ERROR with raises(ValueError) as excinfo: numpy_pickle.dump(data, fname, compress='lz4') excinfo.match(msg) with raises(ValueError) as excinfo: numpy_pickle.dump(data, fname + '.lz4') excinfo.match(msg) protocols = [pickle.DEFAULT_PROTOCOL] if pickle.HIGHEST_PROTOCOL != pickle.DEFAULT_PROTOCOL: protocols.append(pickle.HIGHEST_PROTOCOL) @with_numpy @parametrize('protocol', protocols) def test_memmap_alignment_padding(tmpdir, protocol): # Test that memmaped arrays returned by numpy.load are correctly aligned fname = tmpdir.join('test.mmap').strpath a = np.random.randn(2) numpy_pickle.dump(a, fname, protocol=protocol) memmap = numpy_pickle.load(fname, mmap_mode='r') assert isinstance(memmap, np.memmap) np.testing.assert_array_equal(a, memmap) assert ( memmap.ctypes.data % numpy_pickle.NUMPY_ARRAY_ALIGNMENT_BYTES == 0) assert memmap.flags.aligned array_list = [ np.random.randn(2), np.random.randn(2), np.random.randn(2), np.random.randn(2) ] # On Windows OSError 22 if reusing the same path for memmap ... fname = tmpdir.join('test1.mmap').strpath numpy_pickle.dump(array_list, fname, protocol=protocol) l_reloaded = numpy_pickle.load(fname, mmap_mode='r') for idx, memmap in enumerate(l_reloaded): assert isinstance(memmap, np.memmap) np.testing.assert_array_equal(array_list[idx], memmap) assert ( memmap.ctypes.data % numpy_pickle.NUMPY_ARRAY_ALIGNMENT_BYTES == 0) assert memmap.flags.aligned array_dict = { 'a0': np.arange(2, dtype=np.uint8), 'a1': np.arange(3, dtype=np.uint8), 'a2': np.arange(5, dtype=np.uint8), 'a3': np.arange(7, dtype=np.uint8), 'a4': np.arange(11, dtype=np.uint8), 'a5': np.arange(13, dtype=np.uint8), 'a6': np.arange(17, dtype=np.uint8), 'a7': np.arange(19, dtype=np.uint8), 'a8': np.arange(23, dtype=np.uint8), } # On Windows OSError 22 if reusing the same path for memmap ... fname = tmpdir.join('test2.mmap').strpath numpy_pickle.dump(array_dict, fname, protocol=protocol) d_reloaded = numpy_pickle.load(fname, mmap_mode='r') for key, memmap in d_reloaded.items(): assert isinstance(memmap, np.memmap) np.testing.assert_array_equal(array_dict[key], memmap) assert ( memmap.ctypes.data % numpy_pickle.NUMPY_ARRAY_ALIGNMENT_BYTES == 0) assert memmap.flags.aligned joblib-1.3.2/joblib/test/test_numpy_pickle_compat.py000066400000000000000000000011411446465525000226610ustar00rootroot00000000000000"""Test the old numpy pickler, compatibility version.""" # numpy_pickle is not a drop-in replacement of pickle, as it takes # filenames instead of open files as arguments. from joblib import numpy_pickle_compat def test_z_file(tmpdir): # Test saving and loading data with Zfiles. filename = tmpdir.join('test.pkl').strpath data = numpy_pickle_compat.asbytes('Foo, \n Bar, baz, \n\nfoobar') with open(filename, 'wb') as f: numpy_pickle_compat.write_zfile(f, data) with open(filename, 'rb') as f: data_read = numpy_pickle_compat.read_zfile(f) assert data == data_read joblib-1.3.2/joblib/test/test_numpy_pickle_utils.py000066400000000000000000000005771446465525000225520ustar00rootroot00000000000000from joblib.compressor import BinaryZlibFile from joblib.testing import parametrize @parametrize('filename', ['test', u'test']) # testing str and unicode names def test_binary_zlib_file(tmpdir, filename): """Testing creation of files depending on the type of the filenames.""" binary_file = BinaryZlibFile(tmpdir.join(filename).strpath, mode='wb') binary_file.close() joblib-1.3.2/joblib/test/test_parallel.py000066400000000000000000002056421446465525000204270ustar00rootroot00000000000000""" Test the parallel module. """ # Author: Gael Varoquaux # Copyright (c) 2010-2011 Gael Varoquaux # License: BSD Style, 3 clauses. import os import sys import time import mmap import weakref import warnings import threading from traceback import format_exception from math import sqrt from time import sleep from pickle import PicklingError from contextlib import nullcontext from multiprocessing import TimeoutError import pytest import joblib from joblib import parallel from joblib import dump, load from joblib._multiprocessing_helpers import mp from joblib.test.common import np, with_numpy from joblib.test.common import with_multiprocessing from joblib.test.common import IS_PYPY, force_gc_pypy from joblib.testing import (parametrize, raises, check_subprocess_call, skipif, warns) if mp is not None: # Loky is not available if multiprocessing is not from joblib.externals.loky import get_reusable_executor from queue import Queue try: import posix except ImportError: posix = None try: from ._openmp_test_helper.parallel_sum import parallel_sum except ImportError: parallel_sum = None try: import distributed except ImportError: distributed = None from joblib._parallel_backends import SequentialBackend from joblib._parallel_backends import ThreadingBackend from joblib._parallel_backends import MultiprocessingBackend from joblib._parallel_backends import ParallelBackendBase from joblib._parallel_backends import LokyBackend from joblib.parallel import Parallel, delayed from joblib.parallel import parallel_config from joblib.parallel import parallel_backend from joblib.parallel import register_parallel_backend from joblib.parallel import effective_n_jobs, cpu_count from joblib.parallel import mp, BACKENDS, DEFAULT_BACKEND RETURN_GENERATOR_BACKENDS = BACKENDS.copy() RETURN_GENERATOR_BACKENDS.pop("multiprocessing", None) ALL_VALID_BACKENDS = [None] + sorted(BACKENDS.keys()) # Add instances of backend classes deriving from ParallelBackendBase ALL_VALID_BACKENDS += [BACKENDS[backend_str]() for backend_str in BACKENDS] if mp is None: PROCESS_BACKENDS = [] else: PROCESS_BACKENDS = ['multiprocessing', 'loky'] PARALLEL_BACKENDS = PROCESS_BACKENDS + ['threading'] if hasattr(mp, 'get_context'): # Custom multiprocessing context in Python 3.4+ ALL_VALID_BACKENDS.append(mp.get_context('spawn')) DefaultBackend = BACKENDS[DEFAULT_BACKEND] def get_workers(backend): return getattr(backend, '_pool', getattr(backend, '_workers', None)) def division(x, y): return x / y def square(x): return x ** 2 class MyExceptionWithFinickyInit(Exception): """An exception class with non trivial __init__ """ def __init__(self, a, b, c, d): pass def exception_raiser(x, custom_exception=False): if x == 7: raise (MyExceptionWithFinickyInit('a', 'b', 'c', 'd') if custom_exception else ValueError) return x def interrupt_raiser(x): time.sleep(.05) raise KeyboardInterrupt def f(x, y=0, z=0): """ A module-level function so that it can be spawn with multiprocessing. """ return x ** 2 + y + z def _active_backend_type(): return type(parallel.get_active_backend()[0]) def parallel_func(inner_n_jobs, backend): return Parallel(n_jobs=inner_n_jobs, backend=backend)( delayed(square)(i) for i in range(3)) ############################################################################### def test_cpu_count(): assert cpu_count() > 0 def test_effective_n_jobs(): assert effective_n_jobs() > 0 @parametrize("context", [parallel_config, parallel_backend]) @pytest.mark.parametrize( "backend_n_jobs, expected_n_jobs", [(3, 3), (-1, effective_n_jobs(n_jobs=-1)), (None, 1)], ids=["positive-int", "negative-int", "None"] ) @with_multiprocessing def test_effective_n_jobs_None(context, backend_n_jobs, expected_n_jobs): # check the number of effective jobs when `n_jobs=None` # non-regression test for https://github.com/joblib/joblib/issues/984 with context("threading", n_jobs=backend_n_jobs): # when using a backend, the default of number jobs will be the one set # in the backend assert effective_n_jobs(n_jobs=None) == expected_n_jobs # without any backend, None will default to a single job assert effective_n_jobs(n_jobs=None) == 1 ############################################################################### # Test parallel @parametrize('backend', ALL_VALID_BACKENDS) @parametrize('n_jobs', [1, 2, -1, -2]) @parametrize('verbose', [2, 11, 100]) def test_simple_parallel(backend, n_jobs, verbose): assert ([square(x) for x in range(5)] == Parallel(n_jobs=n_jobs, backend=backend, verbose=verbose)( delayed(square)(x) for x in range(5))) @parametrize('backend', ALL_VALID_BACKENDS) def test_main_thread_renamed_no_warning(backend, monkeypatch): # Check that no default backend relies on the name of the main thread: # https://github.com/joblib/joblib/issues/180#issuecomment-253266247 # Some programs use a different name for the main thread. This is the case # for uWSGI apps for instance. monkeypatch.setattr(target=threading.current_thread(), name='name', value='some_new_name_for_the_main_thread') with warnings.catch_warnings(record=True) as warninfo: results = Parallel(n_jobs=2, backend=backend)( delayed(square)(x) for x in range(3)) assert results == [0, 1, 4] # Due to the default parameters of LokyBackend, there is a chance that # warninfo catches Warnings from worker timeouts. We remove it if it exists warninfo = [w for w in warninfo if "worker timeout" not in str(w.message)] # The multiprocessing backend will raise a warning when detecting that is # started from the non-main thread. Let's check that there is no false # positive because of the name change. assert len(warninfo) == 0 def _assert_warning_nested(backend, inner_n_jobs, expected): with warnings.catch_warnings(record=True) as warninfo: warnings.simplefilter("always") parallel_func(backend=backend, inner_n_jobs=inner_n_jobs) warninfo = [w.message for w in warninfo] if expected: if warninfo: warnings_are_correct = all( 'backed parallel loops cannot' in each.args[0] for each in warninfo ) # With Python nogil, when the outer backend is threading, we might # see more that one warning warnings_have_the_right_length = ( len(warninfo) >= 1 if getattr(sys.flags, 'nogil', False) else len(warninfo) == 1) return warnings_are_correct and warnings_have_the_right_length return False else: assert not warninfo return True @with_multiprocessing @parametrize('parent_backend,child_backend,expected', [ ('loky', 'multiprocessing', True), ('loky', 'loky', False), ('multiprocessing', 'multiprocessing', True), ('multiprocessing', 'loky', True), ('threading', 'multiprocessing', True), ('threading', 'loky', True), ]) def test_nested_parallel_warnings(parent_backend, child_backend, expected): # no warnings if inner_n_jobs=1 Parallel(n_jobs=2, backend=parent_backend)( delayed(_assert_warning_nested)( backend=child_backend, inner_n_jobs=1, expected=False) for _ in range(5)) # warnings if inner_n_jobs != 1 and expected res = Parallel(n_jobs=2, backend=parent_backend)( delayed(_assert_warning_nested)( backend=child_backend, inner_n_jobs=2, expected=expected) for _ in range(5)) # warning handling is not thread safe. One thread might see multiple # warning or no warning at all. if parent_backend == "threading": if IS_PYPY and not any(res): # Related to joblib#1426, should be removed once it is solved. pytest.xfail(reason="This test often fails in PyPy.") assert any(res) else: assert all(res) @with_multiprocessing @parametrize('backend', ['loky', 'multiprocessing', 'threading']) def test_background_thread_parallelism(backend): is_run_parallel = [False] def background_thread(is_run_parallel): with warnings.catch_warnings(record=True) as warninfo: Parallel(n_jobs=2)( delayed(sleep)(.1) for _ in range(4)) print(len(warninfo)) is_run_parallel[0] = len(warninfo) == 0 t = threading.Thread(target=background_thread, args=(is_run_parallel,)) t.start() t.join() assert is_run_parallel[0] def nested_loop(backend): Parallel(n_jobs=2, backend=backend)( delayed(square)(.01) for _ in range(2)) @parametrize('child_backend', BACKENDS) @parametrize('parent_backend', BACKENDS) def test_nested_loop(parent_backend, child_backend): Parallel(n_jobs=2, backend=parent_backend)( delayed(nested_loop)(child_backend) for _ in range(2)) def raise_exception(backend): raise ValueError @with_multiprocessing def test_nested_loop_with_exception_with_loky(): with raises(ValueError): with Parallel(n_jobs=2, backend="loky") as parallel: parallel([delayed(nested_loop)("loky"), delayed(raise_exception)("loky")]) def test_mutate_input_with_threads(): """Input is mutable when using the threading backend""" q = Queue(maxsize=5) Parallel(n_jobs=2, backend="threading")( delayed(q.put)(1) for _ in range(5)) assert q.full() @parametrize('n_jobs', [1, 2, 3]) def test_parallel_kwargs(n_jobs): """Check the keyword argument processing of pmap.""" lst = range(10) assert ([f(x, y=1) for x in lst] == Parallel(n_jobs=n_jobs)(delayed(f)(x, y=1) for x in lst)) @parametrize('backend', PARALLEL_BACKENDS) def test_parallel_as_context_manager(backend): lst = range(10) expected = [f(x, y=1) for x in lst] with Parallel(n_jobs=4, backend=backend) as p: # Internally a pool instance has been eagerly created and is managed # via the context manager protocol managed_backend = p._backend # We make call with the managed parallel object several times inside # the managed block: assert expected == p(delayed(f)(x, y=1) for x in lst) assert expected == p(delayed(f)(x, y=1) for x in lst) # Those calls have all used the same pool instance: if mp is not None: assert get_workers(managed_backend) is get_workers(p._backend) # As soon as we exit the context manager block, the pool is terminated and # no longer referenced from the parallel object: if mp is not None: assert get_workers(p._backend) is None # It's still possible to use the parallel instance in non-managed mode: assert expected == p(delayed(f)(x, y=1) for x in lst) if mp is not None: assert get_workers(p._backend) is None @with_multiprocessing def test_parallel_pickling(): """ Check that pmap captures the errors when it is passed an object that cannot be pickled. """ class UnpicklableObject(object): def __reduce__(self): raise RuntimeError('123') with raises(PicklingError, match=r"the task to send"): Parallel(n_jobs=2, backend='loky')(delayed(id)( UnpicklableObject()) for _ in range(10)) @parametrize('backend', PARALLEL_BACKENDS) def test_parallel_timeout_success(backend): # Check that timeout isn't thrown when function is fast enough assert len(Parallel(n_jobs=2, backend=backend, timeout=30)( delayed(sleep)(0.001) for x in range(10))) == 10 @with_multiprocessing @parametrize('backend', PARALLEL_BACKENDS) def test_parallel_timeout_fail(backend): # Check that timeout properly fails when function is too slow with raises(TimeoutError): Parallel(n_jobs=2, backend=backend, timeout=0.01)( delayed(sleep)(10) for x in range(10)) @with_multiprocessing @parametrize('backend', PROCESS_BACKENDS) def test_error_capture(backend): # Check that error are captured, and that correct exceptions # are raised. if mp is not None: with raises(ZeroDivisionError): Parallel(n_jobs=2, backend=backend)( [delayed(division)(x, y) for x, y in zip((0, 1), (1, 0))]) with raises(KeyboardInterrupt): Parallel(n_jobs=2, backend=backend)( [delayed(interrupt_raiser)(x) for x in (1, 0)]) # Try again with the context manager API with Parallel(n_jobs=2, backend=backend) as parallel: assert get_workers(parallel._backend) is not None original_workers = get_workers(parallel._backend) with raises(ZeroDivisionError): parallel([delayed(division)(x, y) for x, y in zip((0, 1), (1, 0))]) # The managed pool should still be available and be in a working # state despite the previously raised (and caught) exception assert get_workers(parallel._backend) is not None # The pool should have been interrupted and restarted: assert get_workers(parallel._backend) is not original_workers assert ([f(x, y=1) for x in range(10)] == parallel(delayed(f)(x, y=1) for x in range(10))) original_workers = get_workers(parallel._backend) with raises(KeyboardInterrupt): parallel([delayed(interrupt_raiser)(x) for x in (1, 0)]) # The pool should still be available despite the exception assert get_workers(parallel._backend) is not None # The pool should have been interrupted and restarted: assert get_workers(parallel._backend) is not original_workers assert ([f(x, y=1) for x in range(10)] == parallel(delayed(f)(x, y=1) for x in range(10))), ( parallel._iterating, parallel.n_completed_tasks, parallel.n_dispatched_tasks, parallel._aborting ) # Check that the inner pool has been terminated when exiting the # context manager assert get_workers(parallel._backend) is None else: with raises(KeyboardInterrupt): Parallel(n_jobs=2)( [delayed(interrupt_raiser)(x) for x in (1, 0)]) # wrapped exceptions should inherit from the class of the original # exception to make it easy to catch them with raises(ZeroDivisionError): Parallel(n_jobs=2)( [delayed(division)(x, y) for x, y in zip((0, 1), (1, 0))]) with raises(MyExceptionWithFinickyInit): Parallel(n_jobs=2, verbose=0)( (delayed(exception_raiser)(i, custom_exception=True) for i in range(30))) def consumer(queue, item): queue.append('Consumed %s' % item) @parametrize('backend', BACKENDS) @parametrize('batch_size, expected_queue', [(1, ['Produced 0', 'Consumed 0', 'Produced 1', 'Consumed 1', 'Produced 2', 'Consumed 2', 'Produced 3', 'Consumed 3', 'Produced 4', 'Consumed 4', 'Produced 5', 'Consumed 5']), (4, [ # First Batch 'Produced 0', 'Produced 1', 'Produced 2', 'Produced 3', 'Consumed 0', 'Consumed 1', 'Consumed 2', 'Consumed 3', # Second batch 'Produced 4', 'Produced 5', 'Consumed 4', 'Consumed 5'])]) def test_dispatch_one_job(backend, batch_size, expected_queue): """ Test that with only one job, Parallel does act as a iterator. """ queue = list() def producer(): for i in range(6): queue.append('Produced %i' % i) yield i Parallel(n_jobs=1, batch_size=batch_size, backend=backend)( delayed(consumer)(queue, x) for x in producer()) assert queue == expected_queue assert len(queue) == 12 @with_multiprocessing @parametrize('backend', PARALLEL_BACKENDS) def test_dispatch_multiprocessing(backend): """ Check that using pre_dispatch Parallel does indeed dispatch items lazily. """ manager = mp.Manager() queue = manager.list() def producer(): for i in range(6): queue.append('Produced %i' % i) yield i Parallel(n_jobs=2, batch_size=1, pre_dispatch=3, backend=backend)( delayed(consumer)(queue, 'any') for _ in producer()) queue_contents = list(queue) assert queue_contents[0] == 'Produced 0' # Only 3 tasks are pre-dispatched out of 6. The 4th task is dispatched only # after any of the first 3 jobs have completed. first_consumption_index = queue_contents[:4].index('Consumed any') assert first_consumption_index > -1 produced_3_index = queue_contents.index('Produced 3') # 4th task produced assert produced_3_index > first_consumption_index assert len(queue) == 12 def test_batching_auto_threading(): # batching='auto' with the threading backend leaves the effective batch # size to 1 (no batching) as it has been found to never be beneficial with # this low-overhead backend. with Parallel(n_jobs=2, batch_size='auto', backend='threading') as p: p(delayed(id)(i) for i in range(5000)) # many very fast tasks assert p._backend.compute_batch_size() == 1 @with_multiprocessing @parametrize('backend', PROCESS_BACKENDS) def test_batching_auto_subprocesses(backend): with Parallel(n_jobs=2, batch_size='auto', backend=backend) as p: p(delayed(id)(i) for i in range(5000)) # many very fast tasks # It should be strictly larger than 1 but as we don't want heisen # failures on clogged CI worker environment be safe and only check that # it's a strictly positive number. assert p._backend.compute_batch_size() > 0 def test_exception_dispatch(): """Make sure that exception raised during dispatch are indeed captured""" with raises(ValueError): Parallel(n_jobs=2, pre_dispatch=16, verbose=0)( delayed(exception_raiser)(i) for i in range(30)) def nested_function_inner(i): Parallel(n_jobs=2)( delayed(exception_raiser)(j) for j in range(30)) def nested_function_outer(i): Parallel(n_jobs=2)( delayed(nested_function_inner)(j) for j in range(30)) @with_multiprocessing @parametrize('backend', PARALLEL_BACKENDS) @pytest.mark.xfail(reason="https://github.com/joblib/loky/pull/255") def test_nested_exception_dispatch(backend): """Ensure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. """ with raises(ValueError) as excinfo: Parallel(n_jobs=2, backend=backend)( delayed(nested_function_outer)(i) for i in range(30)) # Check that important information such as function names are visible # in the final error message reported to the user report_lines = format_exception(excinfo.type, excinfo.value, excinfo.tb) report = "".join(report_lines) assert 'nested_function_outer' in report assert 'nested_function_inner' in report assert 'exception_raiser' in report assert type(excinfo.value) is ValueError class FakeParallelBackend(SequentialBackend): """Pretends to run concurrently while running sequentially.""" def configure(self, n_jobs=1, parallel=None, **backend_args): self.n_jobs = self.effective_n_jobs(n_jobs) self.parallel = parallel return n_jobs def effective_n_jobs(self, n_jobs=1): if n_jobs < 0: n_jobs = max(mp.cpu_count() + 1 + n_jobs, 1) return n_jobs def test_invalid_backend(): with raises(ValueError, match="Invalid backend:"): Parallel(backend='unit-testing') with raises(ValueError, match="Invalid backend:"): with parallel_config(backend='unit-testing'): pass with raises(ValueError, match="Invalid backend:"): with parallel_config(backend='unit-testing'): pass @parametrize('backend', ALL_VALID_BACKENDS) def test_invalid_njobs(backend): with raises(ValueError) as excinfo: Parallel(n_jobs=0, backend=backend)._initialize_backend() assert "n_jobs == 0 in Parallel has no meaning" in str(excinfo.value) def test_register_parallel_backend(): try: register_parallel_backend("test_backend", FakeParallelBackend) assert "test_backend" in BACKENDS assert BACKENDS["test_backend"] == FakeParallelBackend finally: del BACKENDS["test_backend"] def test_overwrite_default_backend(): assert _active_backend_type() == DefaultBackend try: register_parallel_backend("threading", BACKENDS["threading"], make_default=True) assert _active_backend_type() == ThreadingBackend finally: # Restore the global default manually parallel.DEFAULT_BACKEND = DEFAULT_BACKEND assert _active_backend_type() == DefaultBackend @skipif(mp is not None, reason="Only without multiprocessing") def test_backend_no_multiprocessing(): with warns(UserWarning, match="joblib backend '.*' is not available on.*"): Parallel(backend='loky')(delayed(square)(i) for i in range(3)) # The below should now work without problems with parallel_config(backend='loky'): Parallel()(delayed(square)(i) for i in range(3)) def check_backend_context_manager(context, backend_name): with context(backend_name, n_jobs=3): active_backend, active_n_jobs = parallel.get_active_backend() assert active_n_jobs == 3 assert effective_n_jobs(3) == 3 p = Parallel() assert p.n_jobs == 3 if backend_name == 'multiprocessing': assert type(active_backend) is MultiprocessingBackend assert type(p._backend) is MultiprocessingBackend elif backend_name == 'loky': assert type(active_backend) is LokyBackend assert type(p._backend) is LokyBackend elif backend_name == 'threading': assert type(active_backend) is ThreadingBackend assert type(p._backend) is ThreadingBackend elif backend_name.startswith('test_'): assert type(active_backend) is FakeParallelBackend assert type(p._backend) is FakeParallelBackend all_backends_for_context_manager = PARALLEL_BACKENDS[:] all_backends_for_context_manager.extend( ['test_backend_%d' % i for i in range(3)] ) @with_multiprocessing @parametrize('backend', all_backends_for_context_manager) @parametrize('context', [parallel_backend, parallel_config]) def test_backend_context_manager(monkeypatch, backend, context): if backend not in BACKENDS: monkeypatch.setitem(BACKENDS, backend, FakeParallelBackend) assert _active_backend_type() == DefaultBackend # check that this possible to switch parallel backends sequentially check_backend_context_manager(context, backend) # The default backend is restored assert _active_backend_type() == DefaultBackend # Check that context manager switching is thread safe: Parallel(n_jobs=2, backend='threading')( delayed(check_backend_context_manager)(context, b) for b in all_backends_for_context_manager if not b) # The default backend is again restored assert _active_backend_type() == DefaultBackend class ParameterizedParallelBackend(SequentialBackend): """Pretends to run conncurrently while running sequentially.""" def __init__(self, param=None): if param is None: raise ValueError('param should not be None') self.param = param @parametrize("context", [parallel_config, parallel_backend]) def test_parameterized_backend_context_manager(monkeypatch, context): monkeypatch.setitem(BACKENDS, 'param_backend', ParameterizedParallelBackend) assert _active_backend_type() == DefaultBackend with context('param_backend', param=42, n_jobs=3): active_backend, active_n_jobs = parallel.get_active_backend() assert type(active_backend) is ParameterizedParallelBackend assert active_backend.param == 42 assert active_n_jobs == 3 p = Parallel() assert p.n_jobs == 3 assert p._backend is active_backend results = p(delayed(sqrt)(i) for i in range(5)) assert results == [sqrt(i) for i in range(5)] # The default backend is again restored assert _active_backend_type() == DefaultBackend @parametrize("context", [parallel_config, parallel_backend]) def test_directly_parameterized_backend_context_manager(context): assert _active_backend_type() == DefaultBackend # Check that it's possible to pass a backend instance directly, # without registration with context(ParameterizedParallelBackend(param=43), n_jobs=5): active_backend, active_n_jobs = parallel.get_active_backend() assert type(active_backend) is ParameterizedParallelBackend assert active_backend.param == 43 assert active_n_jobs == 5 p = Parallel() assert p.n_jobs == 5 assert p._backend is active_backend results = p(delayed(sqrt)(i) for i in range(5)) assert results == [sqrt(i) for i in range(5)] # The default backend is again restored assert _active_backend_type() == DefaultBackend def sleep_and_return_pid(): sleep(.1) return os.getpid() def get_nested_pids(): assert _active_backend_type() == ThreadingBackend # Assert that the nested backend does not change the default number of # jobs used in Parallel assert Parallel()._effective_n_jobs() == 1 # Assert that the tasks are running only on one process return Parallel(n_jobs=2)(delayed(sleep_and_return_pid)() for _ in range(2)) class MyBackend(joblib._parallel_backends.LokyBackend): """Backend to test backward compatibility with older backends""" def get_nested_backend(self, ): # Older backends only return a backend, without n_jobs indications. return super(MyBackend, self).get_nested_backend()[0] register_parallel_backend('back_compat_backend', MyBackend) @with_multiprocessing @parametrize('backend', ['threading', 'loky', 'multiprocessing', 'back_compat_backend']) @parametrize("context", [parallel_config, parallel_backend]) def test_nested_backend_context_manager(context, backend): # Check that by default, nested parallel calls will always use the # ThreadingBackend with context(backend): pid_groups = Parallel(n_jobs=2)( delayed(get_nested_pids)() for _ in range(10) ) for pid_group in pid_groups: assert len(set(pid_group)) == 1 @with_multiprocessing @parametrize('n_jobs', [2, -1, None]) @parametrize('backend', PARALLEL_BACKENDS) @parametrize("context", [parallel_config, parallel_backend]) def test_nested_backend_in_sequential(backend, n_jobs, context): # Check that by default, nested parallel calls will always use the # ThreadingBackend def check_nested_backend(expected_backend_type, expected_n_job): # Assert that the sequential backend at top level, does not change the # backend for nested calls. assert _active_backend_type() == BACKENDS[expected_backend_type] # Assert that the nested backend in SequentialBackend does not change # the default number of jobs used in Parallel expected_n_job = effective_n_jobs(expected_n_job) assert Parallel()._effective_n_jobs() == expected_n_job Parallel(n_jobs=1)( delayed(check_nested_backend)(DEFAULT_BACKEND, 1) for _ in range(10) ) with context(backend, n_jobs=n_jobs): Parallel(n_jobs=1)( delayed(check_nested_backend)(backend, n_jobs) for _ in range(10) ) def check_nesting_level(context, inner_backend, expected_level): with context(inner_backend) as ctx: if context is parallel_config: backend = ctx["backend"] if context is parallel_backend: backend = ctx[0] assert backend.nesting_level == expected_level @with_multiprocessing @parametrize('outer_backend', PARALLEL_BACKENDS) @parametrize('inner_backend', PARALLEL_BACKENDS) @parametrize("context", [parallel_config, parallel_backend]) def test_backend_nesting_level(context, outer_backend, inner_backend): # Check that the nesting level for the backend is correctly set check_nesting_level(context, outer_backend, 0) Parallel(n_jobs=2, backend=outer_backend)( delayed(check_nesting_level)(context, inner_backend, 1) for _ in range(10) ) with context(inner_backend, n_jobs=2): Parallel()(delayed(check_nesting_level)(context, inner_backend, 1) for _ in range(10)) @with_multiprocessing @parametrize("context", [parallel_config, parallel_backend]) @parametrize('with_retrieve_callback', [True, False]) def test_retrieval_context(context, with_retrieve_callback): import contextlib class MyBackend(ThreadingBackend): i = 0 supports_retrieve_callback = with_retrieve_callback @contextlib.contextmanager def retrieval_context(self): self.i += 1 yield register_parallel_backend("retrieval", MyBackend) def nested_call(n): return Parallel(n_jobs=2)(delayed(id)(i) for i in range(n)) with context("retrieval") as ctx: Parallel(n_jobs=2)( delayed(nested_call)(i) for i in range(5) ) if context is parallel_config: assert ctx["backend"].i == 1 if context is parallel_backend: assert ctx[0].i == 1 ############################################################################### # Test helpers @parametrize('batch_size', [0, -1, 1.42]) def test_invalid_batch_size(batch_size): with raises(ValueError): Parallel(batch_size=batch_size) @parametrize('n_tasks, n_jobs, pre_dispatch, batch_size', [(2, 2, 'all', 'auto'), (2, 2, 'n_jobs', 'auto'), (10, 2, 'n_jobs', 'auto'), (517, 2, 'n_jobs', 'auto'), (10, 2, 'n_jobs', 'auto'), (10, 4, 'n_jobs', 'auto'), (200, 12, 'n_jobs', 'auto'), (25, 12, '2 * n_jobs', 1), (250, 12, 'all', 1), (250, 12, '2 * n_jobs', 7), (200, 12, '2 * n_jobs', 'auto')]) def test_dispatch_race_condition(n_tasks, n_jobs, pre_dispatch, batch_size): # Check that using (async-)dispatch does not yield a race condition on the # iterable generator that is not thread-safe natively. # This is a non-regression test for the "Pool seems closed" class of error params = {'n_jobs': n_jobs, 'pre_dispatch': pre_dispatch, 'batch_size': batch_size} expected = [square(i) for i in range(n_tasks)] results = Parallel(**params)(delayed(square)(i) for i in range(n_tasks)) assert results == expected @with_multiprocessing def test_default_mp_context(): mp_start_method = mp.get_start_method() p = Parallel(n_jobs=2, backend='multiprocessing') context = p._backend_args.get('context') start_method = context.get_start_method() assert start_method == mp_start_method @with_numpy @with_multiprocessing @parametrize('backend', PROCESS_BACKENDS) def test_no_blas_crash_or_freeze_with_subprocesses(backend): if backend == 'multiprocessing': # Use the spawn backend that is both robust and available on all # platforms backend = mp.get_context('spawn') # Check that on recent Python version, the 'spawn' start method can make # it possible to use multiprocessing in conjunction of any BLAS # implementation that happens to be used by numpy with causing a freeze or # a crash rng = np.random.RandomState(42) # call BLAS DGEMM to force the initialization of the internal thread-pool # in the main process a = rng.randn(1000, 1000) np.dot(a, a.T) # check that the internal BLAS thread-pool is not in an inconsistent state # in the worker processes managed by multiprocessing Parallel(n_jobs=2, backend=backend)( delayed(np.dot)(a, a.T) for i in range(2)) UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAIN = """\ from joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) """ @with_multiprocessing @parametrize('backend', PROCESS_BACKENDS) def test_parallel_with_interactively_defined_functions(backend): # When using the "-c" flag, interactive functions defined in __main__ # should work with any backend. if backend == "multiprocessing" and mp.get_start_method() != "fork": pytest.skip("Require fork start method to use interactively defined " "functions with multiprocessing.") code = UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAIN.format(backend) check_subprocess_call( [sys.executable, '-c', code], timeout=10, stdout_regex=r'\[0, 1, 4, 9, 16\]') UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAIN = """\ import sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) """ SQUARE_MAIN = """\ def square(x): return x ** 2 """ SQUARE_LOCAL = """\ def gen_square(): def square(x): return x ** 2 return square square = gen_square() """ SQUARE_LAMBDA = """\ square = lambda x: x ** 2 """ @with_multiprocessing @parametrize('backend', PROCESS_BACKENDS + ([] if mp is None else ['spawn'])) @parametrize('define_func', [SQUARE_MAIN, SQUARE_LOCAL, SQUARE_LAMBDA]) @parametrize('callable_position', ['delayed', 'args', 'kwargs']) def test_parallel_with_unpicklable_functions_in_args( backend, define_func, callable_position, tmpdir): if backend in ['multiprocessing', 'spawn'] and ( define_func != SQUARE_MAIN or sys.platform == "win32"): pytest.skip("Not picklable with pickle") code = UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAIN.format( define_func=define_func, backend=backend, callable_position=callable_position, joblib_root_folder=os.path.dirname(os.path.dirname(joblib.__file__))) code_file = tmpdir.join("unpicklable_func_script.py") code_file.write(code) check_subprocess_call( [sys.executable, code_file.strpath], timeout=10, stdout_regex=r'\[0, 1, 4, 9, 16\]') INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENT = """\ import sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) """.format(joblib_root_folder=os.path.dirname( os.path.dirname(joblib.__file__))) @with_multiprocessing def test_parallel_with_interactively_defined_functions_loky(tmpdir): # loky accepts interactive functions defined in __main__ and does not # require if __name__ == '__main__' even when the __main__ module is # defined by the result of the execution of a filesystem script. script = tmpdir.join('joblib_interactively_defined_function.py') script.write(INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENT) check_subprocess_call( [sys.executable, script.strpath], stdout_regex=r'\[0, 1, 4, 9, 16\]', timeout=None, # rely on faulthandler to kill the process ) INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENT = """\ import sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) """.format(joblib_root_folder=os.path.dirname( os.path.dirname(joblib.__file__))) @with_multiprocessing def test_parallel_with_interactively_defined_bound_method_loky(tmpdir): script = tmpdir.join('joblib_interactive_bound_method_script.py') script.write(INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENT) check_subprocess_call([sys.executable, script.strpath], stdout_regex=r'\[None, None, None\]', stderr_regex=r'LokyProcess', timeout=15) def test_parallel_with_exhausted_iterator(): exhausted_iterator = iter([]) assert Parallel(n_jobs=2)(exhausted_iterator) == [] def _cleanup_worker(): """Helper function to force gc in each worker.""" force_gc_pypy() time.sleep(.1) def check_memmap(a): if not isinstance(a, np.memmap): raise TypeError('Expected np.memmap instance, got %r', type(a)) return a.copy() # return a regular array instead of a memmap @with_numpy @with_multiprocessing @parametrize('backend', PROCESS_BACKENDS) def test_auto_memmap_on_arrays_from_generator(backend): # Non-regression test for a problem with a bad interaction between the # GC collecting arrays recently created during iteration inside the # parallel dispatch loop and the auto-memmap feature of Parallel. # See: https://github.com/joblib/joblib/pull/294 def generate_arrays(n): for i in range(n): yield np.ones(10, dtype=np.float32) * i # Use max_nbytes=1 to force the use of memory-mapping even for small # arrays results = Parallel(n_jobs=2, max_nbytes=1, backend=backend)( delayed(check_memmap)(a) for a in generate_arrays(100)) for result, expected in zip(results, generate_arrays(len(results))): np.testing.assert_array_equal(expected, result) # Second call to force loky to adapt the executor by growing the number # of worker processes. This is a non-regression test for: # https://github.com/joblib/joblib/issues/629. results = Parallel(n_jobs=4, max_nbytes=1, backend=backend)( delayed(check_memmap)(a) for a in generate_arrays(100)) for result, expected in zip(results, generate_arrays(len(results))): np.testing.assert_array_equal(expected, result) def identity(arg): return arg @with_numpy @with_multiprocessing def test_memmap_with_big_offset(tmpdir): fname = tmpdir.join('test.mmap').strpath size = mmap.ALLOCATIONGRANULARITY obj = [np.zeros(size, dtype='uint8'), np.ones(size, dtype='uint8')] dump(obj, fname) memmap = load(fname, mmap_mode='r') result, = Parallel(n_jobs=2)(delayed(identity)(memmap) for _ in [0]) assert isinstance(memmap[1], np.memmap) assert memmap[1].offset > size np.testing.assert_array_equal(obj, result) def test_warning_about_timeout_not_supported_by_backend(): with warnings.catch_warnings(record=True) as warninfo: Parallel(n_jobs=1, timeout=1)(delayed(square)(i) for i in range(50)) assert len(warninfo) == 1 w = warninfo[0] assert isinstance(w.message, UserWarning) assert str(w.message) == ( "The backend class 'SequentialBackend' does not support timeout. " "You have set 'timeout=1' in Parallel but the 'timeout' parameter " "will not be used.") def set_list_value(input_list, index, value): input_list[index] = value return value @pytest.mark.parametrize('n_jobs', [1, 2, 4]) def test_parallel_return_order_with_return_as_generator_parameter(n_jobs): # This test inserts values in a list in some expected order # in sequential computing, and then checks that this order has been # respected by Parallel output generator. input_list = [0] * 5 result = Parallel(n_jobs=n_jobs, return_as="generator", backend='threading')( delayed(set_list_value)(input_list, i, i) for i in range(5)) # Ensure that all the tasks are completed before checking the result result = list(result) assert all(v == r for v, r in zip(input_list, result)) @parametrize('backend', ALL_VALID_BACKENDS) @parametrize('n_jobs', [1, 2, -2, -1]) def test_abort_backend(n_jobs, backend): delays = ["a"] + [10] * 100 with raises(TypeError): t_start = time.time() Parallel(n_jobs=n_jobs, backend=backend)( delayed(time.sleep)(i) for i in delays) dt = time.time() - t_start assert dt < 20 def get_large_object(arg): result = np.ones(int(5 * 1e5), dtype=bool) result[0] = False return result @with_numpy @parametrize('backend', RETURN_GENERATOR_BACKENDS) @parametrize('n_jobs', [1, 2, -2, -1]) def test_deadlock_with_generator(backend, n_jobs): # Non-regression test for a race condition in the backends when the pickler # is delayed by a large object. with Parallel(n_jobs=n_jobs, backend=backend, return_as="generator") as parallel: result = parallel(delayed(get_large_object)(i) for i in range(10)) next(result) next(result) del result # The gc in pypy can be delayed. Force it to make sure this test does # not cause timeout on the CI. force_gc_pypy() @parametrize('backend', RETURN_GENERATOR_BACKENDS) @parametrize('n_jobs', [1, 2, -2, -1]) def test_multiple_generator_call(backend, n_jobs): # Non-regression test that ensures the dispatch of the tasks starts # immediately when Parallel.__call__ is called. This test relies on the # assumption that only one generator can be submitted at a time. with raises(RuntimeError, match="This Parallel instance is already running"): parallel = Parallel(n_jobs, backend=backend, return_as="generator") g = parallel(delayed(sleep)(1) for _ in range(10)) # noqa: F841 t_start = time.time() gen2 = parallel(delayed(id)(i) for i in range(100)) # noqa: F841 # Make sure that the error is raised quickly assert time.time() - t_start < 2, ( "The error should be raised immediatly when submitting a new task " "but it took more than 2s." ) del g # The gc in pypy can be delayed. Force it to make sure this test does not # cause timeout on the CI. force_gc_pypy() @parametrize('backend', RETURN_GENERATOR_BACKENDS) @parametrize('n_jobs', [1, 2, -2, -1]) def test_multiple_generator_call_managed(backend, n_jobs): # Non-regression test that ensures the dispatch of the tasks starts # immediately when Parallel.__call__ is called. This test relies on the # assumption that only one generator can be submitted at a time. with Parallel(n_jobs, backend=backend, return_as="generator") as parallel: g = parallel(delayed(sleep)(10) for _ in range(10)) # noqa: F841 t_start = time.time() with raises(RuntimeError, match="This Parallel instance is already running"): g2 = parallel(delayed(id)(i) for i in range(100)) # noqa: F841 # Make sure that the error is raised quickly assert time.time() - t_start < 2, ( "The error should be raised immediatly when submitting a new task " "but it took more than 2s." ) # The gc in pypy can be delayed. Force it to make sure this test does not # cause timeout on the CI. del g force_gc_pypy() @parametrize('backend', RETURN_GENERATOR_BACKENDS) @parametrize('n_jobs', [1, 2, -2, -1]) def test_multiple_generator_call_separated(backend, n_jobs): # Check that for separated Parallel, both tasks are correctly returned. g = Parallel(n_jobs, backend=backend, return_as="generator")( delayed(sqrt)(i ** 2) for i in range(10) ) g2 = Parallel(n_jobs, backend=backend, return_as="generator")( delayed(sqrt)(i ** 2) for i in range(10, 20) ) assert all(res == i for res, i in zip(g, range(10))) assert all(res == i for res, i in zip(g2, range(10, 20))) @parametrize('backend, error', [ ('loky', True), ('threading', False), ('sequential', False), ]) def test_multiple_generator_call_separated_gc(backend, error): if (backend == 'loky') and (mp is None): pytest.skip("Requires multiprocessing") # Check that in loky, only one call can be run at a time with # a single executor. parallel = Parallel(2, backend=backend, return_as="generator") g = parallel(delayed(sleep)(10) for i in range(10)) g_wr = weakref.finalize(g, lambda: print("Generator collected")) ctx = ( raises(RuntimeError, match="The executor underlying Parallel") if error else nullcontext() ) with ctx: # For loky, this call will raise an error as the gc of the previous # generator will shutdown the shared executor. # For the other backends, as the worker pools are not shared between # the two calls, this should proceed correctly. t_start = time.time() g = Parallel(2, backend=backend, return_as="generator")( delayed(sqrt)(i ** 2) for i in range(10, 20) ) # The gc in pypy can be delayed. Force it to test the behavior when it # will eventually be collected. force_gc_pypy() assert all(res == i for res, i in zip(g, range(10, 20))) assert time.time() - t_start < 5 # Make sure that the computation are stopped for the gc'ed generator retry = 0 while g_wr.alive and retry < 3: retry += 1 time.sleep(.5) assert time.time() - t_start < 5 if parallel._effective_n_jobs() != 1: # check that the first parallel object is aborting (the final _aborted # state might be delayed). assert parallel._aborting @with_numpy @with_multiprocessing @parametrize('backend', PROCESS_BACKENDS) def test_memmapping_leaks(backend, tmpdir): # Non-regression test for memmapping backends. Ensure that the data # does not stay too long in memory tmpdir = tmpdir.strpath # Use max_nbytes=1 to force the use of memory-mapping even for small # arrays with Parallel(n_jobs=2, max_nbytes=1, backend=backend, temp_folder=tmpdir) as p: p(delayed(check_memmap)(a) for a in [np.random.random(10)] * 2) # The memmap folder should not be clean in the context scope assert len(os.listdir(tmpdir)) > 0 # Cleaning of the memmap folder is triggered by the garbage # collection. With pypy the garbage collection has been observed to be # delayed, sometimes up until the shutdown of the interpreter. This # cleanup job executed in the worker ensures that it's triggered # immediately. p(delayed(_cleanup_worker)() for _ in range(2)) # Make sure that the shared memory is cleaned at the end when we exit # the context for _ in range(100): if not os.listdir(tmpdir): break sleep(.1) else: raise AssertionError('temporary directory of Parallel was not removed') # Make sure that the shared memory is cleaned at the end of a call p = Parallel(n_jobs=2, max_nbytes=1, backend=backend) p(delayed(check_memmap)(a) for a in [np.random.random(10)] * 2) p(delayed(_cleanup_worker)() for _ in range(2)) for _ in range(100): if not os.listdir(tmpdir): break sleep(.1) else: raise AssertionError('temporary directory of Parallel was not removed') @parametrize('backend', ([None, 'threading'] if mp is None else [None, 'loky', 'threading']) ) def test_lambda_expression(backend): # cloudpickle is used to pickle delayed callables results = Parallel(n_jobs=2, backend=backend)( delayed(lambda x: x ** 2)(i) for i in range(10)) assert results == [i ** 2 for i in range(10)] @with_multiprocessing @parametrize('backend', PROCESS_BACKENDS) def test_backend_batch_statistics_reset(backend): """Test that a parallel backend correctly resets its batch statistics.""" n_jobs = 2 n_inputs = 500 task_time = 2. / n_inputs p = Parallel(verbose=10, n_jobs=n_jobs, backend=backend) p(delayed(time.sleep)(task_time) for i in range(n_inputs)) assert (p._backend._effective_batch_size == p._backend._DEFAULT_EFFECTIVE_BATCH_SIZE) assert (p._backend._smoothed_batch_duration == p._backend._DEFAULT_SMOOTHED_BATCH_DURATION) p(delayed(time.sleep)(task_time) for i in range(n_inputs)) assert (p._backend._effective_batch_size == p._backend._DEFAULT_EFFECTIVE_BATCH_SIZE) assert (p._backend._smoothed_batch_duration == p._backend._DEFAULT_SMOOTHED_BATCH_DURATION) @with_multiprocessing @parametrize("context", [parallel_config, parallel_backend]) def test_backend_hinting_and_constraints(context): for n_jobs in [1, 2, -1]: assert type(Parallel(n_jobs=n_jobs)._backend) == DefaultBackend p = Parallel(n_jobs=n_jobs, prefer='threads') assert type(p._backend) is ThreadingBackend p = Parallel(n_jobs=n_jobs, prefer='processes') assert type(p._backend) is DefaultBackend p = Parallel(n_jobs=n_jobs, require='sharedmem') assert type(p._backend) is ThreadingBackend # Explicit backend selection can override backend hinting although it # is useless to pass a hint when selecting a backend. p = Parallel(n_jobs=2, backend='loky', prefer='threads') assert type(p._backend) is LokyBackend with context('loky', n_jobs=2): # Explicit backend selection by the user with the context manager # should be respected when combined with backend hints only. p = Parallel(prefer='threads') assert type(p._backend) is LokyBackend assert p.n_jobs == 2 with context('loky', n_jobs=2): # Locally hard-coded n_jobs value is respected. p = Parallel(n_jobs=3, prefer='threads') assert type(p._backend) is LokyBackend assert p.n_jobs == 3 with context('loky', n_jobs=2): # Explicit backend selection by the user with the context manager # should be ignored when the Parallel call has hard constraints. # In this case, the default backend that supports shared mem is # used an the default number of processes is used. p = Parallel(require='sharedmem') assert type(p._backend) is ThreadingBackend assert p.n_jobs == 1 with context('loky', n_jobs=2): p = Parallel(n_jobs=3, require='sharedmem') assert type(p._backend) is ThreadingBackend assert p.n_jobs == 3 @parametrize("context", [parallel_config, parallel_backend]) def test_backend_hinting_and_constraints_with_custom_backends( capsys, context ): # Custom backends can declare that they use threads and have shared memory # semantics: class MyCustomThreadingBackend(ParallelBackendBase): supports_sharedmem = True use_threads = True def apply_async(self): pass def effective_n_jobs(self, n_jobs): return n_jobs with context(MyCustomThreadingBackend()): p = Parallel(n_jobs=2, prefer='processes') # ignored assert type(p._backend) is MyCustomThreadingBackend p = Parallel(n_jobs=2, require='sharedmem') assert type(p._backend) is MyCustomThreadingBackend class MyCustomProcessingBackend(ParallelBackendBase): supports_sharedmem = False use_threads = False def apply_async(self): pass def effective_n_jobs(self, n_jobs): return n_jobs with context(MyCustomProcessingBackend()): p = Parallel(n_jobs=2, prefer='processes') assert type(p._backend) is MyCustomProcessingBackend out, err = capsys.readouterr() assert out == "" assert err == "" p = Parallel(n_jobs=2, require='sharedmem', verbose=10) assert type(p._backend) is ThreadingBackend out, err = capsys.readouterr() expected = ("Using ThreadingBackend as joblib backend " "instead of MyCustomProcessingBackend as the latter " "does not provide shared memory semantics.") assert out.strip() == expected assert err == "" with raises(ValueError): Parallel(backend=MyCustomProcessingBackend(), require='sharedmem') def test_invalid_backend_hinting_and_constraints(): with raises(ValueError): Parallel(prefer='invalid') with raises(ValueError): Parallel(require='invalid') with raises(ValueError): # It is inconsistent to prefer process-based parallelism while # requiring shared memory semantics. Parallel(prefer='processes', require='sharedmem') if mp is not None: # It is inconsistent to ask explicitly for a process-based # parallelism while requiring shared memory semantics. with raises(ValueError): Parallel(backend='loky', require='sharedmem') with raises(ValueError): Parallel(backend='multiprocessing', require='sharedmem') def _recursive_backend_info(limit=3, **kwargs): """Perform nested parallel calls and introspect the backend on the way""" with Parallel(n_jobs=2) as p: this_level = [(type(p._backend).__name__, p._backend.nesting_level)] if limit == 0: return this_level results = p(delayed(_recursive_backend_info)(limit=limit - 1, **kwargs) for i in range(1)) return this_level + results[0] @with_multiprocessing @parametrize('backend', ['loky', 'threading']) @parametrize("context", [parallel_config, parallel_backend]) def test_nested_parallelism_limit(context, backend): with context(backend, n_jobs=2): backend_types_and_levels = _recursive_backend_info() if cpu_count() == 1: second_level_backend_type = 'SequentialBackend' max_level = 1 else: second_level_backend_type = 'ThreadingBackend' max_level = 2 top_level_backend_type = backend.title() + 'Backend' expected_types_and_levels = [ (top_level_backend_type, 0), (second_level_backend_type, 1), ('SequentialBackend', max_level), ('SequentialBackend', max_level) ] assert backend_types_and_levels == expected_types_and_levels @with_numpy @skipif(distributed is None, reason='This test requires dask') @parametrize("context", [parallel_config, parallel_backend]) def test_nested_parallelism_with_dask(context): client = distributed.Client(n_workers=2, threads_per_worker=2) # noqa # 10 MB of data as argument to trigger implicit scattering data = np.ones(int(1e7), dtype=np.uint8) for i in range(2): with context('dask'): backend_types_and_levels = _recursive_backend_info(data=data) assert len(backend_types_and_levels) == 4 assert all(name == 'DaskDistributedBackend' for name, _ in backend_types_and_levels) # No argument with context('dask'): backend_types_and_levels = _recursive_backend_info() assert len(backend_types_and_levels) == 4 assert all(name == 'DaskDistributedBackend' for name, _ in backend_types_and_levels) def _recursive_parallel(nesting_limit=None): """A horrible function that does recursive parallel calls""" return Parallel()(delayed(_recursive_parallel)() for i in range(2)) @pytest.mark.no_cover @parametrize("context", [parallel_config, parallel_backend]) @parametrize( 'backend', (['threading'] if mp is None else ['loky', 'threading']) ) def test_thread_bomb_mitigation(context, backend): # Test that recursive parallelism raises a recursion rather than # saturating the operating system resources by creating a unbounded number # of threads. with context(backend, n_jobs=2): with raises(BaseException) as excinfo: _recursive_parallel() exc = excinfo.value if backend == "loky": # Local import because loky may not be importable for lack of # multiprocessing from joblib.externals.loky.process_executor import TerminatedWorkerError # noqa if isinstance(exc, (TerminatedWorkerError, PicklingError)): # The recursion exception can itself cause an error when # pickling it to be send back to the parent process. In this # case the worker crashes but the original traceback is still # printed on stderr. This could be improved but does not seem # simple to do and this is not critical for users (as long # as there is no process or thread bomb happening). pytest.xfail("Loky worker crash when serializing RecursionError") assert isinstance(exc, RecursionError) def _run_parallel_sum(): env_vars = {} for var in ['OMP_NUM_THREADS', 'OPENBLAS_NUM_THREADS', 'MKL_NUM_THREADS', 'VECLIB_MAXIMUM_THREADS', 'NUMEXPR_NUM_THREADS', 'NUMBA_NUM_THREADS', 'ENABLE_IPC']: env_vars[var] = os.environ.get(var) return env_vars, parallel_sum(100) @parametrize("backend", ([None, 'loky'] if mp is not None else [None])) @skipif(parallel_sum is None, reason="Need OpenMP helper compiled") def test_parallel_thread_limit(backend): results = Parallel(n_jobs=2, backend=backend)( delayed(_run_parallel_sum)() for _ in range(2) ) expected_num_threads = max(cpu_count() // 2, 1) for worker_env_vars, omp_num_threads in results: assert omp_num_threads == expected_num_threads for name, value in worker_env_vars.items(): if name.endswith("_THREADS"): assert value == str(expected_num_threads) else: assert name == "ENABLE_IPC" assert value == "1" @skipif(distributed is not None, reason='This test requires dask NOT installed') @parametrize("context", [parallel_config, parallel_backend]) def test_dask_backend_when_dask_not_installed(context): with raises(ValueError, match='Please install dask'): context('dask') @parametrize("context", [parallel_config, parallel_backend]) def test_zero_worker_backend(context): # joblib.Parallel should reject with an explicit error message parallel # backends that have no worker. class ZeroWorkerBackend(ThreadingBackend): def configure(self, *args, **kwargs): return 0 def apply_async(self, func, callback=None): # pragma: no cover raise TimeoutError("No worker available") def effective_n_jobs(self, n_jobs): # pragma: no cover return 0 expected_msg = "ZeroWorkerBackend has no active worker" with context(ZeroWorkerBackend()): with pytest.raises(RuntimeError, match=expected_msg): Parallel(n_jobs=2)(delayed(id)(i) for i in range(2)) def test_globals_update_at_each_parallel_call(): # This is a non-regression test related to joblib issues #836 and #833. # Cloudpickle versions between 0.5.4 and 0.7 introduced a bug where global # variables changes in a parent process between two calls to # joblib.Parallel would not be propagated into the workers. global MY_GLOBAL_VARIABLE MY_GLOBAL_VARIABLE = "original value" def check_globals(): global MY_GLOBAL_VARIABLE return MY_GLOBAL_VARIABLE assert check_globals() == "original value" workers_global_variable = Parallel(n_jobs=2)( delayed(check_globals)() for i in range(2)) assert set(workers_global_variable) == {"original value"} # Change the value of MY_GLOBAL_VARIABLE, and make sure this change gets # propagated into the workers environment MY_GLOBAL_VARIABLE = "changed value" assert check_globals() == "changed value" workers_global_variable = Parallel(n_jobs=2)( delayed(check_globals)() for i in range(2)) assert set(workers_global_variable) == {"changed value"} ############################################################################## # Test environment variable in child env, in particular for limiting # the maximal number of threads in C-library threadpools. # def _check_numpy_threadpool_limits(): import numpy as np # Let's call BLAS on a Matrix Matrix multiplication with dimensions large # enough to ensure that the threadpool managed by the underlying BLAS # implementation is actually used so as to force its initialization. a = np.random.randn(100, 100) np.dot(a, a) from threadpoolctl import threadpool_info return threadpool_info() def _parent_max_num_threads_for(child_module, parent_info): for parent_module in parent_info: if parent_module['filepath'] == child_module['filepath']: return parent_module['num_threads'] raise ValueError("An unexpected module was loaded in child:\n{}" .format(child_module)) def check_child_num_threads(workers_info, parent_info, num_threads): # Check that the number of threads reported in workers_info is consistent # with the expectation. We need to be careful to handle the cases where # the requested number of threads is below max_num_thread for the library. for child_threadpool_info in workers_info: for child_module in child_threadpool_info: parent_max_num_threads = _parent_max_num_threads_for( child_module, parent_info) expected = {min(num_threads, parent_max_num_threads), num_threads} assert child_module['num_threads'] in expected @with_numpy @with_multiprocessing @parametrize('n_jobs', [2, 4, -2, -1]) def test_threadpool_limitation_in_child_loky(n_jobs): # Check that the protection against oversubscription in workers is working # using threadpoolctl functionalities. # Skip this test if numpy is not linked to a BLAS library parent_info = _check_numpy_threadpool_limits() if len(parent_info) == 0: pytest.skip(msg="Need a version of numpy linked to BLAS") workers_threadpool_infos = Parallel(backend="loky", n_jobs=n_jobs)( delayed(_check_numpy_threadpool_limits)() for i in range(2)) n_jobs = effective_n_jobs(n_jobs) expected_child_num_threads = max(cpu_count() // n_jobs, 1) check_child_num_threads(workers_threadpool_infos, parent_info, expected_child_num_threads) @with_numpy @with_multiprocessing @parametrize('inner_max_num_threads', [1, 2, 4, None]) @parametrize('n_jobs', [2, -1]) @parametrize("context", [parallel_config, parallel_backend]) def test_threadpool_limitation_in_child_context( context, n_jobs, inner_max_num_threads ): # Check that the protection against oversubscription in workers is working # using threadpoolctl functionalities. # Skip this test if numpy is not linked to a BLAS library parent_info = _check_numpy_threadpool_limits() if len(parent_info) == 0: pytest.skip(msg="Need a version of numpy linked to BLAS") with context('loky', inner_max_num_threads=inner_max_num_threads): workers_threadpool_infos = Parallel(n_jobs=n_jobs)( delayed(_check_numpy_threadpool_limits)() for i in range(2)) n_jobs = effective_n_jobs(n_jobs) if inner_max_num_threads is None: expected_child_num_threads = max(cpu_count() // n_jobs, 1) else: expected_child_num_threads = inner_max_num_threads check_child_num_threads(workers_threadpool_infos, parent_info, expected_child_num_threads) @with_multiprocessing @parametrize('n_jobs', [2, -1]) @parametrize('var_name', ["OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "OMP_NUM_THREADS"]) @parametrize("context", [parallel_config, parallel_backend]) def test_threadpool_limitation_in_child_override(context, n_jobs, var_name): # Check that environment variables set by the user on the main process # always have the priority. # Clean up the existing executor because we change the environment of the # parent at runtime and it is not detected in loky intentionally. get_reusable_executor(reuse=True).shutdown() def _get_env(var_name): return os.environ.get(var_name) original_var_value = os.environ.get(var_name) try: os.environ[var_name] = "4" # Skip this test if numpy is not linked to a BLAS library results = Parallel(n_jobs=n_jobs)( delayed(_get_env)(var_name) for i in range(2)) assert results == ["4", "4"] with context('loky', inner_max_num_threads=1): results = Parallel(n_jobs=n_jobs)( delayed(_get_env)(var_name) for i in range(2)) assert results == ["1", "1"] finally: if original_var_value is None: del os.environ[var_name] else: os.environ[var_name] = original_var_value @with_multiprocessing @parametrize('n_jobs', [2, 4, -1]) def test_loky_reuse_workers(n_jobs): # Non-regression test for issue #967 where the workers are not reused when # calling multiple Parallel loops. def parallel_call(n_jobs): x = range(10) Parallel(n_jobs=n_jobs)(delayed(sum)(x) for i in range(10)) # Run a parallel loop and get the workers used for computations parallel_call(n_jobs) first_executor = get_reusable_executor(reuse=True) # Ensure that the workers are reused for the next calls, as the executor is # not restarted. for _ in range(10): parallel_call(n_jobs) executor = get_reusable_executor(reuse=True) assert executor == first_executor joblib-1.3.2/joblib/test/test_store_backends.py000066400000000000000000000060611446465525000216130ustar00rootroot00000000000000 try: # Python 2.7: use the C pickle to speed up # test_concurrency_safe_write which pickles big python objects import cPickle as cpickle except ImportError: import pickle as cpickle import functools from pickle import PicklingError import time import pytest from joblib.testing import parametrize, timeout from joblib.test.common import with_multiprocessing from joblib.backports import concurrency_safe_rename from joblib import Parallel, delayed from joblib._store_backends import ( concurrency_safe_write, FileSystemStoreBackend, CacheWarning, ) def write_func(output, filename): with open(filename, 'wb') as f: cpickle.dump(output, f) def load_func(expected, filename): for i in range(10): try: with open(filename, 'rb') as f: reloaded = cpickle.load(f) break except (OSError, IOError): # On Windows you can have WindowsError ([Error 5] Access # is denied or [Error 13] Permission denied) when reading the file, # probably because a writer process has a lock on the file time.sleep(0.1) else: raise assert expected == reloaded def concurrency_safe_write_rename(to_write, filename, write_func): temporary_filename = concurrency_safe_write(to_write, filename, write_func) concurrency_safe_rename(temporary_filename, filename) @timeout(0) # No timeout as this test can be long @with_multiprocessing @parametrize('backend', ['multiprocessing', 'loky', 'threading']) def test_concurrency_safe_write(tmpdir, backend): # Add one item to cache filename = tmpdir.join('test.pkl').strpath obj = {str(i): i for i in range(int(1e5))} funcs = [functools.partial(concurrency_safe_write_rename, write_func=write_func) if i % 3 != 2 else load_func for i in range(12)] Parallel(n_jobs=2, backend=backend)( delayed(func)(obj, filename) for func in funcs) def test_warning_on_dump_failure(tmpdir): # Check that a warning is raised when the dump fails for any reason but # a PicklingError. class UnpicklableObject(object): def __reduce__(self): raise RuntimeError("some exception") backend = FileSystemStoreBackend() backend.location = tmpdir.join('test_warning_on_pickling_error').strpath backend.compress = None with pytest.warns(CacheWarning, match="some exception"): backend.dump_item("testpath", UnpicklableObject()) def test_warning_on_pickling_error(tmpdir): # This is separate from test_warning_on_dump_failure because in the # future we will turn this into an exception. class UnpicklableObject(object): def __reduce__(self): raise PicklingError("not picklable") backend = FileSystemStoreBackend() backend.location = tmpdir.join('test_warning_on_pickling_error').strpath backend.compress = None with pytest.warns(FutureWarning, match="not picklable"): backend.dump_item("testpath", UnpicklableObject()) joblib-1.3.2/joblib/test/test_testing.py000066400000000000000000000050121446465525000202750ustar00rootroot00000000000000import sys import re from joblib.testing import raises, check_subprocess_call def test_check_subprocess_call(): code = '\n'.join(['result = 1 + 2 * 3', 'print(result)', 'my_list = [1, 2, 3]', 'print(my_list)']) check_subprocess_call([sys.executable, '-c', code]) # Now checking stdout with a regex check_subprocess_call([sys.executable, '-c', code], # Regex needed for platform-specific line endings stdout_regex=r'7\s{1,2}\[1, 2, 3\]') def test_check_subprocess_call_non_matching_regex(): code = '42' non_matching_pattern = '_no_way_this_matches_anything_' with raises(ValueError) as excinfo: check_subprocess_call([sys.executable, '-c', code], stdout_regex=non_matching_pattern) excinfo.match('Unexpected stdout.+{}'.format(non_matching_pattern)) def test_check_subprocess_call_wrong_command(): wrong_command = '_a_command_that_does_not_exist_' with raises(OSError): check_subprocess_call([wrong_command]) def test_check_subprocess_call_non_zero_return_code(): code_with_non_zero_exit = '\n'.join([ 'import sys', 'print("writing on stdout")', 'sys.stderr.write("writing on stderr")', 'sys.exit(123)']) pattern = re.compile('Non-zero return code: 123.+' 'Stdout:\nwriting on stdout.+' 'Stderr:\nwriting on stderr', re.DOTALL) with raises(ValueError) as excinfo: check_subprocess_call([sys.executable, '-c', code_with_non_zero_exit]) excinfo.match(pattern) def test_check_subprocess_call_timeout(): code_timing_out = '\n'.join([ 'import time', 'import sys', 'print("before sleep on stdout")', 'sys.stdout.flush()', 'sys.stderr.write("before sleep on stderr")', 'sys.stderr.flush()', # We need to sleep for at least 2 * timeout seconds in case the SIGKILL # is triggered. 'time.sleep(10)', 'print("process should have be killed before")', 'sys.stdout.flush()']) pattern = re.compile('Non-zero return code:.+' 'Stdout:\nbefore sleep on stdout\\s+' 'Stderr:\nbefore sleep on stderr', re.DOTALL) with raises(ValueError) as excinfo: check_subprocess_call([sys.executable, '-c', code_timing_out], timeout=1) excinfo.match(pattern) joblib-1.3.2/joblib/test/test_utils.py000066400000000000000000000011101446465525000177530ustar00rootroot00000000000000import pytest from joblib._utils import eval_expr @pytest.mark.parametrize( "expr", ["exec('import os')", "print(1)", "import os", "1+1; import os", "1^1"], ) def test_eval_expr_invalid(expr): with pytest.raises( ValueError, match="is not a valid or supported arithmetic" ): eval_expr(expr) @pytest.mark.parametrize( "expr, result", [ ("2*6", 12), ("2**6", 64), ("1 + 2*3**(4) / (6 + -7)", -161.0), ("(20 // 3) % 5", 1), ], ) def test_eval_expr_valid(expr, result): assert eval_expr(expr) == result joblib-1.3.2/joblib/test/testutils.py000066400000000000000000000003731446465525000176260ustar00rootroot00000000000000def return_slice_of_data(arr, start_idx, end_idx): return arr[start_idx:end_idx] def print_filename_and_raise(arr): from joblib._memmapping_reducer import _get_backing_memmap print(_get_backing_memmap(arr).filename) raise ValueError joblib-1.3.2/joblib/testing.py000066400000000000000000000060251446465525000162640ustar00rootroot00000000000000""" Helper for testing. """ import sys import warnings import os.path import re import subprocess import threading import pytest import _pytest raises = pytest.raises warns = pytest.warns SkipTest = _pytest.runner.Skipped skipif = pytest.mark.skipif fixture = pytest.fixture parametrize = pytest.mark.parametrize timeout = pytest.mark.timeout xfail = pytest.mark.xfail param = pytest.param def warnings_to_stdout(): """ Redirect all warnings to stdout. """ showwarning_orig = warnings.showwarning def showwarning(msg, cat, fname, lno, file=None, line=0): showwarning_orig(msg, cat, os.path.basename(fname), line, sys.stdout) warnings.showwarning = showwarning # warnings.simplefilter('always') def check_subprocess_call(cmd, timeout=5, stdout_regex=None, stderr_regex=None): """Runs a command in a subprocess with timeout in seconds. A SIGTERM is sent after `timeout` and if it does not terminate, a SIGKILL is sent after `2 * timeout`. Also checks returncode is zero, stdout if stdout_regex is set, and stderr if stderr_regex is set. """ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) def terminate_process(): # pragma: no cover """ Attempt to terminate a leftover process spawned during test execution: ideally this should not be needed but can help avoid clogging the CI workers in case of deadlocks. """ warnings.warn(f"Timeout running {cmd}") proc.terminate() def kill_process(): # pragma: no cover """ Kill a leftover process spawned during test execution: ideally this should not be needed but can help avoid clogging the CI workers in case of deadlocks. """ warnings.warn(f"Timeout running {cmd}") proc.kill() try: if timeout is not None: terminate_timer = threading.Timer(timeout, terminate_process) terminate_timer.start() kill_timer = threading.Timer(2 * timeout, kill_process) kill_timer.start() stdout, stderr = proc.communicate() stdout, stderr = stdout.decode(), stderr.decode() if proc.returncode != 0: message = ( 'Non-zero return code: {}.\nStdout:\n{}\n' 'Stderr:\n{}').format( proc.returncode, stdout, stderr) raise ValueError(message) if (stdout_regex is not None and not re.search(stdout_regex, stdout)): raise ValueError( "Unexpected stdout: {!r} does not match:\n{!r}".format( stdout_regex, stdout)) if (stderr_regex is not None and not re.search(stderr_regex, stderr)): raise ValueError( "Unexpected stderr: {!r} does not match:\n{!r}".format( stderr_regex, stderr)) finally: if timeout is not None: terminate_timer.cancel() kill_timer.cancel() joblib-1.3.2/pyproject.toml000066400000000000000000000044451446465525000157140ustar00rootroot00000000000000[build-system] requires = ["setuptools>=61.2"] build-backend = "setuptools.build_meta" [project] name = "joblib" authors = [{name = "Gael Varoquaux", email = "gael.varoquaux@normalesup.org"}] license = {text = "BSD 3-Clause"} description = "Lightweight pipelining with Python functions" classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Intended Audience :: Education", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering", "Topic :: Utilities", "Topic :: Software Development :: Libraries", ] requires-python = ">=3.7" dynamic = ["version"] [project.readme] file = "README.rst" content-type = "text/x-rst" [project.urls] Homepage = "https://joblib.readthedocs.io" Source = "https://github.com/joblib/joblib" [tool.setuptools] packages = [ "joblib", "joblib.test", "joblib.test.data", "joblib.externals", "joblib.externals.cloudpickle", "joblib.externals.loky", "joblib.externals.loky.backend", ] platforms = ["any"] include-package-data = false [tool.setuptools.package-data] "joblib.test" = [ "data/*.gz", "data/*.gzip", "data/*.bz2", "data/*.xz", "data/*.lzma", "data/*.pkl", "data/*.npy", "data/*.npy.z", ] [tool.setuptools.dynamic] version = {attr = "joblib.__version__"} [aliases] release = "egg_info -RDb ''" # Make sure the docs are uploaded when we do an upload upload = "upload upload_docs --upload-dir doc/_build/html" [bdist_rpm] doc_files = "doc" [tool.pytest.ini_options] doctest_optionflags = [ "NORMALIZE_WHITESPACE", "ELLIPSIS" ] addopts = [ "--doctest-glob='doc/*.rst'", "--doctest-modules", "--color=yes", ] testpaths = "joblib" norecursedirs = "joblib/externals/*" [tool.coverage.run] omit = [ "joblib/test/data/*", "joblib/test/_openmp_test_helper/setup.py", "*/joblib/externals/*", ] [tool.coverage.report] show_missing = true joblib-1.3.2/setup.cfg000066400000000000000000000004461446465525000146160ustar00rootroot00000000000000# The prefered config file is pyproject.toml. The use of setup.cfg is # mostly for compatibility with flake8 so it should not be used if possible. [flake8] exclude= joblib/externals/*, doc/auto_examples, doc/_build, per-file-ignores = examples/*: E402, doc/conf.py: E402, joblib-1.3.2/setup.py000077500000000000000000000001341446465525000145040ustar00rootroot00000000000000#!/usr/bin/env python from setuptools import setup if __name__ == "__main__": setup()