Upstream: Not applicable Reason: Use system loky and cloudpickle diff --git a/joblib/_cloudpickle_wrapper.py b/joblib/_cloudpickle_wrapper.py index daf899d..2305f34 100644 --- a/joblib/_cloudpickle_wrapper.py +++ b/joblib/_cloudpickle_wrapper.py @@ -12,7 +12,7 @@ def _my_wrap_non_picklable_objects(obj, keep_wrapper=True): if mp is not None: - from .externals.loky import wrap_non_picklable_objects + from loky import wrap_non_picklable_objects else: wrap_non_picklable_objects = _my_wrap_non_picklable_objects diff --git a/joblib/_memmapping_reducer.py b/joblib/_memmapping_reducer.py index 283399c..7479d61 100644 --- a/joblib/_memmapping_reducer.py +++ b/joblib/_memmapping_reducer.py @@ -34,7 +34,7 @@ except ImportError: from .numpy_pickle import dump, load, load_temporary_memmap from .backports import make_memmap from .disk import delete_folder -from .externals.loky.backend import resource_tracker +from loky.backend import resource_tracker # Some system have a ramdisk mounted by default, we can use it instead of /tmp # as the default folder to dump big arrays to share with subprocesses. @@ -58,7 +58,7 @@ JOBLIB_MMAPS = set() def _log_and_unlink(filename): - from .externals.loky.backend.resource_tracker import _resource_tracker + from loky.backend.resource_tracker import _resource_tracker util.debug( "[FINALIZER CALL] object mapping to {} about to be deleted," " decrementing the refcount of the file (pid: {})".format( diff --git a/joblib/_parallel_backends.py b/joblib/_parallel_backends.py index 1852b0d..73fa682 100644 --- a/joblib/_parallel_backends.py +++ b/joblib/_parallel_backends.py @@ -18,9 +18,9 @@ if mp is not None: from .executor import get_memmapping_executor # Import loky only if multiprocessing is present - from .externals.loky import process_executor, cpu_count - from .externals.loky.process_executor import ShutdownExecutorError - from .externals.loky.process_executor import _ExceptionWithTraceback + from loky import process_executor, cpu_count + from loky.process_executor import ShutdownExecutorError + from loky.process_executor import _ExceptionWithTraceback class ParallelBackendBase(metaclass=ABCMeta): diff --git a/joblib/executor.py b/joblib/executor.py index 6837a7d..6a22f18 100644 --- a/joblib/executor.py +++ b/joblib/executor.py @@ -10,7 +10,7 @@ copy between the parent and child processes. from ._memmapping_reducer import get_memmapping_reducers from ._memmapping_reducer import TemporaryResourcesManager -from .externals.loky.reusable_executor import _ReusablePoolExecutor +from loky.reusable_executor import _ReusablePoolExecutor _executor_args = None diff --git a/joblib/externals/loky/backend/reduction.py b/joblib/externals/loky/backend/reduction.py index bed32ba..69e6485 100644 --- a/joblib/externals/loky/backend/reduction.py +++ b/joblib/externals/loky/backend/reduction.py @@ -80,7 +80,7 @@ else: # global variable to change the pickler behavior try: - from joblib.externals import cloudpickle # noqa: F401 + import cloudpickle # noqa: F401 DEFAULT_ENV = "cloudpickle" except ImportError: @@ -108,7 +108,7 @@ def set_loky_pickler(loky_pickler=None): return if loky_pickler == "cloudpickle": - from joblib.externals.cloudpickle import CloudPickler as loky_pickler_cls + from cloudpickle import CloudPickler as loky_pickler_cls else: try: from importlib import import_module diff --git a/joblib/externals/loky/cloudpickle_wrapper.py b/joblib/externals/loky/cloudpickle_wrapper.py index 099debc..165a524 100644 --- a/joblib/externals/loky/cloudpickle_wrapper.py +++ b/joblib/externals/loky/cloudpickle_wrapper.py @@ -1,6 +1,6 @@ import inspect from functools import partial -from joblib.externals.cloudpickle import dumps, loads +from cloudpickle import dumps, loads WRAP_CACHE = {} diff --git a/joblib/parallel.py b/joblib/parallel.py index dd7d91e..0385e4a 100644 --- a/joblib/parallel.py +++ b/joblib/parallel.py @@ -58,7 +58,7 @@ MAYBE_AVAILABLE_BACKENDS = {'multiprocessing', 'loky'} # backend if mp is not None: BACKENDS['multiprocessing'] = MultiprocessingBackend - from .externals import loky + import loky BACKENDS['loky'] = LokyBackend DEFAULT_BACKEND = 'loky' diff --git a/joblib/test/test_memmapping.py b/joblib/test/test_memmapping.py index 42a297a..548eafa 100644 --- a/joblib/test/test_memmapping.py +++ b/joblib/test/test_memmapping.py @@ -158,7 +158,7 @@ def test_resource_tracker_retries_when_permissionerror(tmpdir): import os import numpy as np import time - from joblib.externals.loky.backend import resource_tracker + from loky.backend import resource_tracker resource_tracker.VERBOSE = 1 # Start the resource tracker @@ -524,7 +524,7 @@ def test_multithreaded_parallel_termination_resource_tracker_silent(): import os import numpy as np from joblib import Parallel, delayed - from joblib.externals.loky.backend import resource_tracker + from loky.backend import resource_tracker from concurrent.futures import ThreadPoolExecutor, wait resource_tracker.VERBOSE = 0 diff --git a/joblib/test/test_module.py b/joblib/test/test_module.py index a2257a4..1a39c08 100644 --- a/joblib/test/test_module.py +++ b/joblib/test/test_module.py @@ -44,7 +44,7 @@ def test_no_semaphore_tracker_on_import(): def test_no_resource_tracker_on_import(): code = """if True: import joblib - from joblib.externals.loky.backend import resource_tracker + from loky.backend import resource_tracker # The following line would raise RuntimeError if the # start_method is already set. msg = "loky.resource_tracker has been spawned on import" diff --git a/joblib/test/test_parallel.py b/joblib/test/test_parallel.py index 9a7ad15..3d15408 100644 --- a/joblib/test/test_parallel.py +++ b/joblib/test/test_parallel.py @@ -35,7 +35,7 @@ from joblib.testing import (parametrize, raises, check_subprocess_call, if mp is not None: # Loky is not available if multiprocessing is not - from joblib.externals.loky import get_reusable_executor + from loky import get_reusable_executor from queue import Queue @@ -1661,7 +1661,7 @@ def test_thread_bomb_mitigation(context, backend): if backend == "loky": # Local import because loky may not be importable for lack of # multiprocessing - from joblib.externals.loky.process_executor import TerminatedWorkerError # noqa + from loky.process_executor import TerminatedWorkerError # noqa if isinstance(exc, (TerminatedWorkerError, PicklingError)): # The recursion exception can itself cause an error when # pickling it to be send back to the parent process. In this