Code reorg utils into utils module reduces complexity (#4990)

* Split apart the massive utils.py into a utils module

news/4992.feature.rst

@@ -0,0 +1 @@
+Internal to pipenv, the utils.py was split into a utils module with unused code removed.

pipenv/cli/command.py

@@ -12,7 +12,7 @@
 )
 from pipenv.exceptions import PipenvOptionsError
 from pipenv.patched import crayons
-from pipenv.utils import subprocess_run
+from pipenv.utils.processes import subprocess_run
 from pipenv.vendor.click import (
     Choice, argument, echo, edit, group, option, pass_context, secho, types,
     version_option
@@ -64,7 +64,7 @@ def cli(
         cleanup_virtualenv, do_clear, do_py, do_where, ensure_project,
         format_help, system_which, warn_in_virtualenv
     )
-    from ..utils import create_spinner
+    from pipenv.utils.spinner import create_spinner
 
     if man:
         if system_which("man"):

pipenv/cli/options.py

@@ -1,7 +1,7 @@
 import os
 
 from pipenv.project import Project
-from pipenv.utils import is_valid_url
+from pipenv.utils.internet import is_valid_url
 from pipenv.vendor.click import (
     BadArgumentUsage, BadParameter, Group, Option, argument, echo,
     make_pass_decorator, option

pipenv/core.py

@@ -15,14 +15,22 @@
 from pipenv import environments, exceptions, pep508checker, progress
 from pipenv._compat import decode_for_output, fix_utf8
 from pipenv.patched import crayons
-from pipenv.utils import (
-    cmd_list_to_shell, convert_deps_to_pip, create_spinner, download_file,
-    find_python, get_canonical_names, get_host_and_port, get_source_list, is_pinned,
-    is_python_command, is_required_version, is_star, is_valid_url,
-    parse_indexes, pep423_name, prepare_pip_source_args, proper_case,
-    python_version, run_command, subprocess_run, venv_resolve_deps
-)
 
+from pipenv.utils.dependencies import (
+    convert_deps_to_pip,
+    get_canonical_names,
+    is_pinned,
+    is_required_version,
+    is_star,
+    pep423_name,
+    python_version
+)
+from pipenv.utils.internet import download_file, get_host_and_port, is_valid_url, proper_case
+from pipenv.utils.indexes import get_source_list, parse_indexes, prepare_pip_source_args
+from pipenv.utils.resolver import venv_resolve_deps
+from pipenv.utils.shell import cmd_list_to_shell, find_python, is_python_command, subprocess_run
+from pipenv.utils.spinner import create_spinner
+from pipenv.utils.processes import run_command
 
 if environments.is_type_checking():
     from typing import Dict, List, Optional, Union
@@ -1017,10 +1025,7 @@ def do_lock(
         for k, v in lockfile[section].copy().items():
             if not hasattr(v, "keys"):
                 del lockfile[section][k]
-    # Ensure that develop inherits from default.
-    dev_packages = project.dev_packages.copy()
-    dev_packages = overwrite_dev(project.packages, dev_packages)
-    # Resolve dev-package dependencies, with pip-tools.
+    # Resolve dev-package dependencies followed by packages dependencies.
     for is_dev in [True, False]:
         pipfile_section = "dev-packages" if is_dev else "packages"
         if project.pipfile_exists:

pipenv/environment.py

@@ -14,7 +14,9 @@
 import pipenv
 
 from pipenv.environments import is_type_checking
-from pipenv.utils import make_posix, normalize_path, subprocess_run
+from pipenv.utils.shell import make_posix, normalize_path
+from pipenv.utils.processes import subprocess_run
+from pipenv.utils.indexes import prepare_pip_source_args
 from pipenv.vendor import vistir
 from pipenv.vendor.cached_property import cached_property
 from pipenv.vendor.packaging.utils import canonicalize_name
@@ -599,7 +601,7 @@ def get_finder(self, pre=False):
         from .vendor.pip_shims.shims import InstallCommand, get_package_finder
 
         pip_command = InstallCommand()
-        pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources)
+        pip_args = prepare_pip_source_args(self.sources)
         pip_options, _ = pip_command.parser.parse_args(pip_args)
         pip_options.cache_dir = self.project.s.PIPENV_CACHE_DIR
         pip_options.pre = self.pipfile.get("pre", pre)

pipenv/installers.py

@@ -5,7 +5,8 @@
 from abc import ABCMeta, abstractmethod
 
 from pipenv.vendor import attr
-from pipenv.utils import find_windows_executable, subprocess_run
+from pipenv.utils.processes import subprocess_run
+from pipenv.utils.shell import find_windows_executable
 
 
 @attr.s

pipenv/project.py

@@ -21,13 +21,20 @@
 from pipenv.core import system_which
 from pipenv.environment import Environment
 from pipenv.environments import Setting, is_type_checking, is_in_virtualenv, normalize_pipfile_path
-from pipenv.utils import (
-    cleanup_toml, convert_toml_outline_tables, find_requirements,
-    find_windows_executable, get_canonical_names, get_pipenv_dist, get_url_name,
-    get_workon_home, is_editable, is_installable_file, is_star, is_valid_url,
-    is_virtual_environment, looks_like_dir, pep423_name,
-    proper_case, python_version, safe_expandvars
+from pipenv.utils.dependencies import get_canonical_names, is_editable, is_installable_file, is_star, python_version
+from pipenv.utils.internet import get_url_name, is_valid_url, proper_case
+from pipenv.utils.resolver import pep423_name
+from pipenv.utils.toml import cleanup_toml, convert_toml_outline_tables
+from pipenv.utils.shell import (
+    find_requirements,
+    find_windows_executable,
+    get_pipenv_dist,
+    get_workon_home,
+    is_virtual_environment,
+    looks_like_dir,
+    safe_expandvars
 )
+
 from pipenv.vendor.cached_property import cached_property
 from pipenv.vendor.requirementslib.models.utils import (
     get_default_pyproject_backend
@@ -285,7 +292,7 @@ def get_location_for_virtualenv(self):
     @property
     def working_set(self):
         # type: () -> pkg_resources.WorkingSet
-        from .utils import load_path
+        from pipenv.utils.shell import load_path
         sys_path = load_path(self.which("python"))
         import pkg_resources
         return pkg_resources.WorkingSet(sys_path)
@@ -902,7 +909,7 @@ def find_source(self, source):
         return source
 
     def get_source(self, name=None, url=None, refresh=False):
-        from .utils import is_url_equal
+        from pipenv.utils.internet import is_url_equal
 
         def find_source(sources, name=None, url=None):
             source = None

pipenv/resolver.py

@@ -585,7 +585,7 @@ def __getattribute__(self, key):
 
 
 def clean_results(results, resolver, project, dev=False):
-    from pipenv.utils import translate_markers
+    from pipenv.utils.dependencies import translate_markers
     if not project.lockfile_exists:
         return results
     lockfile = project.lockfile_content
@@ -646,7 +646,7 @@ def clean_outdated(results, resolver, project, dev=False):
 def parse_packages(packages, pre, clear, system, requirements_dir=None):
     from pipenv.vendor.requirementslib.models.requirements import Requirement
     from pipenv.vendor.vistir.contextmanagers import cd, temp_path
-    from pipenv.utils import parse_indexes
+    from pipenv.utils.indexes import parse_indexes
     parsed_packages = []
     for package in packages:
         *_, line = parse_indexes(package)
@@ -674,7 +674,8 @@ def parse_packages(packages, pre, clear, system, requirements_dir=None):
 
 
 def resolve_packages(pre, clear, verbose, system, write, requirements_dir, packages, dev):
-    from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources
+    from pipenv.utils.internet import create_mirror_source, replace_pypi_sources
+    from pipenv.utils.resolver import resolve_deps
     pypi_mirror_source = (
         create_mirror_source(os.environ["PIPENV_PYPI_MIRROR"])
         if "PIPENV_PYPI_MIRROR" in os.environ