req_command.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436
  1. """Contains the Command base classes that depend on PipSession.
  2. The classes in this module are in a separate module so the commands not
  3. needing download / PackageFinder capability don't unnecessarily import the
  4. PackageFinder machinery and all its vendored dependencies, etc.
  5. """
  6. import logging
  7. import os
  8. from functools import partial
  9. from pip._vendor.six import PY2
  10. from pip._internal.cli import cmdoptions
  11. from pip._internal.cli.base_command import Command
  12. from pip._internal.cli.command_context import CommandContextMixIn
  13. from pip._internal.exceptions import CommandError, PreviousBuildDirError
  14. from pip._internal.index.collector import LinkCollector
  15. from pip._internal.index.package_finder import PackageFinder
  16. from pip._internal.models.selection_prefs import SelectionPreferences
  17. from pip._internal.network.session import PipSession
  18. from pip._internal.operations.prepare import RequirementPreparer
  19. from pip._internal.req.constructors import (
  20. install_req_from_editable,
  21. install_req_from_line,
  22. install_req_from_parsed_requirement,
  23. install_req_from_req_string,
  24. )
  25. from pip._internal.req.req_file import parse_requirements
  26. from pip._internal.self_outdated_check import pip_self_version_check
  27. from pip._internal.utils.temp_dir import tempdir_kinds
  28. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  29. if MYPY_CHECK_RUNNING:
  30. from optparse import Values
  31. from typing import Any, List, Optional, Tuple
  32. from pip._internal.cache import WheelCache
  33. from pip._internal.models.target_python import TargetPython
  34. from pip._internal.req.req_install import InstallRequirement
  35. from pip._internal.req.req_tracker import RequirementTracker
  36. from pip._internal.resolution.base import BaseResolver
  37. from pip._internal.utils.temp_dir import TempDirectory, TempDirectoryTypeRegistry
  38. logger = logging.getLogger(__name__)
  39. class SessionCommandMixin(CommandContextMixIn):
  40. """
  41. A class mixin for command classes needing _build_session().
  42. """
  43. def __init__(self):
  44. # type: () -> None
  45. super(SessionCommandMixin, self).__init__()
  46. self._session = None # Optional[PipSession]
  47. @classmethod
  48. def _get_index_urls(cls, options):
  49. # type: (Values) -> Optional[List[str]]
  50. """Return a list of index urls from user-provided options."""
  51. index_urls = []
  52. if not getattr(options, "no_index", False):
  53. url = getattr(options, "index_url", None)
  54. if url:
  55. index_urls.append(url)
  56. urls = getattr(options, "extra_index_urls", None)
  57. if urls:
  58. index_urls.extend(urls)
  59. # Return None rather than an empty list
  60. return index_urls or None
  61. def get_default_session(self, options):
  62. # type: (Values) -> PipSession
  63. """Get a default-managed session."""
  64. if self._session is None:
  65. self._session = self.enter_context(self._build_session(options))
  66. # there's no type annotation on requests.Session, so it's
  67. # automatically ContextManager[Any] and self._session becomes Any,
  68. # then https://github.com/python/mypy/issues/7696 kicks in
  69. assert self._session is not None
  70. return self._session
  71. def _build_session(self, options, retries=None, timeout=None):
  72. # type: (Values, Optional[int], Optional[int]) -> PipSession
  73. assert not options.cache_dir or os.path.isabs(options.cache_dir)
  74. session = PipSession(
  75. cache=(
  76. os.path.join(options.cache_dir, "http")
  77. if options.cache_dir else None
  78. ),
  79. retries=retries if retries is not None else options.retries,
  80. trusted_hosts=options.trusted_hosts,
  81. index_urls=self._get_index_urls(options),
  82. )
  83. # Handle custom ca-bundles from the user
  84. if options.cert:
  85. session.verify = options.cert
  86. # Handle SSL client certificate
  87. if options.client_cert:
  88. session.cert = options.client_cert
  89. # Handle timeouts
  90. if options.timeout or timeout:
  91. session.timeout = (
  92. timeout if timeout is not None else options.timeout
  93. )
  94. # Handle configured proxies
  95. if options.proxy:
  96. session.proxies = {
  97. "http": options.proxy,
  98. "https": options.proxy,
  99. }
  100. # Determine if we can prompt the user for authentication or not
  101. session.auth.prompting = not options.no_input
  102. return session
  103. class IndexGroupCommand(Command, SessionCommandMixin):
  104. """
  105. Abstract base class for commands with the index_group options.
  106. This also corresponds to the commands that permit the pip version check.
  107. """
  108. def handle_pip_version_check(self, options):
  109. # type: (Values) -> None
  110. """
  111. Do the pip version check if not disabled.
  112. This overrides the default behavior of not doing the check.
  113. """
  114. # Make sure the index_group options are present.
  115. assert hasattr(options, 'no_index')
  116. if options.disable_pip_version_check or options.no_index:
  117. return
  118. # Otherwise, check if we're using the latest version of pip available.
  119. session = self._build_session(
  120. options,
  121. retries=0,
  122. timeout=min(5, options.timeout)
  123. )
  124. with session:
  125. pip_self_version_check(session, options)
  126. KEEPABLE_TEMPDIR_TYPES = [
  127. tempdir_kinds.BUILD_ENV,
  128. tempdir_kinds.EPHEM_WHEEL_CACHE,
  129. tempdir_kinds.REQ_BUILD,
  130. ]
  131. def with_cleanup(func):
  132. # type: (Any) -> Any
  133. """Decorator for common logic related to managing temporary
  134. directories.
  135. """
  136. def configure_tempdir_registry(registry):
  137. # type: (TempDirectoryTypeRegistry) -> None
  138. for t in KEEPABLE_TEMPDIR_TYPES:
  139. registry.set_delete(t, False)
  140. def wrapper(self, options, args):
  141. # type: (RequirementCommand, Values, List[Any]) -> Optional[int]
  142. assert self.tempdir_registry is not None
  143. if options.no_clean:
  144. configure_tempdir_registry(self.tempdir_registry)
  145. try:
  146. return func(self, options, args)
  147. except PreviousBuildDirError:
  148. # This kind of conflict can occur when the user passes an explicit
  149. # build directory with a pre-existing folder. In that case we do
  150. # not want to accidentally remove it.
  151. configure_tempdir_registry(self.tempdir_registry)
  152. raise
  153. return wrapper
  154. class RequirementCommand(IndexGroupCommand):
  155. def __init__(self, *args, **kw):
  156. # type: (Any, Any) -> None
  157. super(RequirementCommand, self).__init__(*args, **kw)
  158. self.cmd_opts.add_option(cmdoptions.no_clean())
  159. @staticmethod
  160. def determine_resolver_variant(options):
  161. # type: (Values) -> str
  162. """Determines which resolver should be used, based on the given options."""
  163. # We didn't want to change things for Python 2, since it's nearly done with
  164. # and we're using performance improvements that only work on Python 3.
  165. if PY2:
  166. if '2020-resolver' in options.features_enabled:
  167. return "2020-resolver"
  168. else:
  169. return "legacy"
  170. if "legacy-resolver" in options.deprecated_features_enabled:
  171. return "legacy"
  172. return "2020-resolver"
  173. @classmethod
  174. def make_requirement_preparer(
  175. cls,
  176. temp_build_dir, # type: TempDirectory
  177. options, # type: Values
  178. req_tracker, # type: RequirementTracker
  179. session, # type: PipSession
  180. finder, # type: PackageFinder
  181. use_user_site, # type: bool
  182. download_dir=None, # type: str
  183. ):
  184. # type: (...) -> RequirementPreparer
  185. """
  186. Create a RequirementPreparer instance for the given parameters.
  187. """
  188. temp_build_dir_path = temp_build_dir.path
  189. assert temp_build_dir_path is not None
  190. resolver_variant = cls.determine_resolver_variant(options)
  191. if resolver_variant == "2020-resolver":
  192. lazy_wheel = 'fast-deps' in options.features_enabled
  193. if lazy_wheel:
  194. logger.warning(
  195. 'pip is using lazily downloaded wheels using HTTP '
  196. 'range requests to obtain dependency information. '
  197. 'This experimental feature is enabled through '
  198. '--use-feature=fast-deps and it is not ready for '
  199. 'production.'
  200. )
  201. else:
  202. lazy_wheel = False
  203. if 'fast-deps' in options.features_enabled:
  204. logger.warning(
  205. 'fast-deps has no effect when used with the legacy resolver.'
  206. )
  207. return RequirementPreparer(
  208. build_dir=temp_build_dir_path,
  209. src_dir=options.src_dir,
  210. download_dir=download_dir,
  211. build_isolation=options.build_isolation,
  212. req_tracker=req_tracker,
  213. session=session,
  214. progress_bar=options.progress_bar,
  215. finder=finder,
  216. require_hashes=options.require_hashes,
  217. use_user_site=use_user_site,
  218. lazy_wheel=lazy_wheel,
  219. )
  220. @classmethod
  221. def make_resolver(
  222. cls,
  223. preparer, # type: RequirementPreparer
  224. finder, # type: PackageFinder
  225. options, # type: Values
  226. wheel_cache=None, # type: Optional[WheelCache]
  227. use_user_site=False, # type: bool
  228. ignore_installed=True, # type: bool
  229. ignore_requires_python=False, # type: bool
  230. force_reinstall=False, # type: bool
  231. upgrade_strategy="to-satisfy-only", # type: str
  232. use_pep517=None, # type: Optional[bool]
  233. py_version_info=None, # type: Optional[Tuple[int, ...]]
  234. ):
  235. # type: (...) -> BaseResolver
  236. """
  237. Create a Resolver instance for the given parameters.
  238. """
  239. make_install_req = partial(
  240. install_req_from_req_string,
  241. isolated=options.isolated_mode,
  242. use_pep517=use_pep517,
  243. )
  244. resolver_variant = cls.determine_resolver_variant(options)
  245. # The long import name and duplicated invocation is needed to convince
  246. # Mypy into correctly typechecking. Otherwise it would complain the
  247. # "Resolver" class being redefined.
  248. if resolver_variant == "2020-resolver":
  249. import pip._internal.resolution.resolvelib.resolver
  250. return pip._internal.resolution.resolvelib.resolver.Resolver(
  251. preparer=preparer,
  252. finder=finder,
  253. wheel_cache=wheel_cache,
  254. make_install_req=make_install_req,
  255. use_user_site=use_user_site,
  256. ignore_dependencies=options.ignore_dependencies,
  257. ignore_installed=ignore_installed,
  258. ignore_requires_python=ignore_requires_python,
  259. force_reinstall=force_reinstall,
  260. upgrade_strategy=upgrade_strategy,
  261. py_version_info=py_version_info,
  262. )
  263. import pip._internal.resolution.legacy.resolver
  264. return pip._internal.resolution.legacy.resolver.Resolver(
  265. preparer=preparer,
  266. finder=finder,
  267. wheel_cache=wheel_cache,
  268. make_install_req=make_install_req,
  269. use_user_site=use_user_site,
  270. ignore_dependencies=options.ignore_dependencies,
  271. ignore_installed=ignore_installed,
  272. ignore_requires_python=ignore_requires_python,
  273. force_reinstall=force_reinstall,
  274. upgrade_strategy=upgrade_strategy,
  275. py_version_info=py_version_info,
  276. )
  277. def get_requirements(
  278. self,
  279. args, # type: List[str]
  280. options, # type: Values
  281. finder, # type: PackageFinder
  282. session, # type: PipSession
  283. ):
  284. # type: (...) -> List[InstallRequirement]
  285. """
  286. Parse command-line arguments into the corresponding requirements.
  287. """
  288. requirements = [] # type: List[InstallRequirement]
  289. for filename in options.constraints:
  290. for parsed_req in parse_requirements(
  291. filename,
  292. constraint=True, finder=finder, options=options,
  293. session=session):
  294. req_to_add = install_req_from_parsed_requirement(
  295. parsed_req,
  296. isolated=options.isolated_mode,
  297. user_supplied=False,
  298. )
  299. requirements.append(req_to_add)
  300. for req in args:
  301. req_to_add = install_req_from_line(
  302. req, None, isolated=options.isolated_mode,
  303. use_pep517=options.use_pep517,
  304. user_supplied=True,
  305. )
  306. requirements.append(req_to_add)
  307. for req in options.editables:
  308. req_to_add = install_req_from_editable(
  309. req,
  310. user_supplied=True,
  311. isolated=options.isolated_mode,
  312. use_pep517=options.use_pep517,
  313. )
  314. requirements.append(req_to_add)
  315. # NOTE: options.require_hashes may be set if --require-hashes is True
  316. for filename in options.requirements:
  317. for parsed_req in parse_requirements(
  318. filename,
  319. finder=finder, options=options, session=session):
  320. req_to_add = install_req_from_parsed_requirement(
  321. parsed_req,
  322. isolated=options.isolated_mode,
  323. use_pep517=options.use_pep517,
  324. user_supplied=True,
  325. )
  326. requirements.append(req_to_add)
  327. # If any requirement has hash options, enable hash checking.
  328. if any(req.has_hash_options for req in requirements):
  329. options.require_hashes = True
  330. if not (args or options.editables or options.requirements):
  331. opts = {'name': self.name}
  332. if options.find_links:
  333. raise CommandError(
  334. 'You must give at least one requirement to {name} '
  335. '(maybe you meant "pip {name} {links}"?)'.format(
  336. **dict(opts, links=' '.join(options.find_links))))
  337. else:
  338. raise CommandError(
  339. 'You must give at least one requirement to {name} '
  340. '(see "pip help {name}")'.format(**opts))
  341. return requirements
  342. @staticmethod
  343. def trace_basic_info(finder):
  344. # type: (PackageFinder) -> None
  345. """
  346. Trace basic information about the provided objects.
  347. """
  348. # Display where finder is looking for packages
  349. search_scope = finder.search_scope
  350. locations = search_scope.get_formatted_locations()
  351. if locations:
  352. logger.info(locations)
  353. def _build_package_finder(
  354. self,
  355. options, # type: Values
  356. session, # type: PipSession
  357. target_python=None, # type: Optional[TargetPython]
  358. ignore_requires_python=None, # type: Optional[bool]
  359. ):
  360. # type: (...) -> PackageFinder
  361. """
  362. Create a package finder appropriate to this requirement command.
  363. :param ignore_requires_python: Whether to ignore incompatible
  364. "Requires-Python" values in links. Defaults to False.
  365. """
  366. link_collector = LinkCollector.create(session, options=options)
  367. selection_prefs = SelectionPreferences(
  368. allow_yanked=True,
  369. format_control=options.format_control,
  370. allow_all_prereleases=options.pre,
  371. prefer_binary=options.prefer_binary,
  372. ignore_requires_python=ignore_requires_python,
  373. )
  374. return PackageFinder.create(
  375. link_collector=link_collector,
  376. selection_prefs=selection_prefs,
  377. target_python=target_python,
  378. )