constructors.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476
  1. """Backing implementation for InstallRequirement's various constructors
  2. The idea here is that these formed a major chunk of InstallRequirement's size
  3. so, moving them and support code dedicated to them outside of that class
  4. helps creates for better understandability for the rest of the code.
  5. These are meant to be used elsewhere within pip to create instances of
  6. InstallRequirement.
  7. """
  8. import logging
  9. import os
  10. import re
  11. from pip._vendor.packaging.markers import Marker
  12. from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
  13. from pip._vendor.packaging.specifiers import Specifier
  14. from pip._vendor.pkg_resources import RequirementParseError, parse_requirements
  15. from pip._internal.exceptions import InstallationError
  16. from pip._internal.models.index import PyPI, TestPyPI
  17. from pip._internal.models.link import Link
  18. from pip._internal.models.wheel import Wheel
  19. from pip._internal.pyproject import make_pyproject_path
  20. from pip._internal.req.req_install import InstallRequirement
  21. from pip._internal.utils.deprecation import deprecated
  22. from pip._internal.utils.filetypes import is_archive_file
  23. from pip._internal.utils.misc import is_installable_dir
  24. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  25. from pip._internal.utils.urls import path_to_url
  26. from pip._internal.vcs import is_url, vcs
  27. if MYPY_CHECK_RUNNING:
  28. from typing import Any, Dict, Optional, Set, Tuple, Union
  29. from pip._internal.req.req_file import ParsedRequirement
  30. __all__ = [
  31. "install_req_from_editable", "install_req_from_line",
  32. "parse_editable"
  33. ]
  34. logger = logging.getLogger(__name__)
  35. operators = Specifier._operators.keys()
  36. def _strip_extras(path):
  37. # type: (str) -> Tuple[str, Optional[str]]
  38. m = re.match(r'^(.+)(\[[^\]]+\])$', path)
  39. extras = None
  40. if m:
  41. path_no_extras = m.group(1)
  42. extras = m.group(2)
  43. else:
  44. path_no_extras = path
  45. return path_no_extras, extras
  46. def convert_extras(extras):
  47. # type: (Optional[str]) -> Set[str]
  48. if not extras:
  49. return set()
  50. return Requirement("placeholder" + extras.lower()).extras
  51. def parse_editable(editable_req):
  52. # type: (str) -> Tuple[Optional[str], str, Set[str]]
  53. """Parses an editable requirement into:
  54. - a requirement name
  55. - an URL
  56. - extras
  57. - editable options
  58. Accepted requirements:
  59. svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir
  60. .[some_extra]
  61. """
  62. url = editable_req
  63. # If a file path is specified with extras, strip off the extras.
  64. url_no_extras, extras = _strip_extras(url)
  65. if os.path.isdir(url_no_extras):
  66. if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
  67. msg = (
  68. 'File "setup.py" not found. Directory cannot be installed '
  69. 'in editable mode: {}'.format(os.path.abspath(url_no_extras))
  70. )
  71. pyproject_path = make_pyproject_path(url_no_extras)
  72. if os.path.isfile(pyproject_path):
  73. msg += (
  74. '\n(A "pyproject.toml" file was found, but editable '
  75. 'mode currently requires a setup.py based build.)'
  76. )
  77. raise InstallationError(msg)
  78. # Treating it as code that has already been checked out
  79. url_no_extras = path_to_url(url_no_extras)
  80. if url_no_extras.lower().startswith('file:'):
  81. package_name = Link(url_no_extras).egg_fragment
  82. if extras:
  83. return (
  84. package_name,
  85. url_no_extras,
  86. Requirement("placeholder" + extras.lower()).extras,
  87. )
  88. else:
  89. return package_name, url_no_extras, set()
  90. for version_control in vcs:
  91. if url.lower().startswith('{}:'.format(version_control)):
  92. url = '{}+{}'.format(version_control, url)
  93. break
  94. if '+' not in url:
  95. raise InstallationError(
  96. '{} is not a valid editable requirement. '
  97. 'It should either be a path to a local project or a VCS URL '
  98. '(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req)
  99. )
  100. vc_type = url.split('+', 1)[0].lower()
  101. if not vcs.get_backend(vc_type):
  102. backends = ", ".join([bends.name + '+URL' for bends in vcs.backends])
  103. error_message = "For --editable={}, " \
  104. "only {} are currently supported".format(
  105. editable_req, backends)
  106. raise InstallationError(error_message)
  107. package_name = Link(url).egg_fragment
  108. if not package_name:
  109. raise InstallationError(
  110. "Could not detect requirement name for '{}', please specify one "
  111. "with #egg=your_package_name".format(editable_req)
  112. )
  113. return package_name, url, set()
  114. def deduce_helpful_msg(req):
  115. # type: (str) -> str
  116. """Returns helpful msg in case requirements file does not exist,
  117. or cannot be parsed.
  118. :params req: Requirements file path
  119. """
  120. msg = ""
  121. if os.path.exists(req):
  122. msg = " The path does exist. "
  123. # Try to parse and check if it is a requirements file.
  124. try:
  125. with open(req, 'r') as fp:
  126. # parse first line only
  127. next(parse_requirements(fp.read()))
  128. msg += (
  129. "The argument you provided "
  130. "({}) appears to be a"
  131. " requirements file. If that is the"
  132. " case, use the '-r' flag to install"
  133. " the packages specified within it."
  134. ).format(req)
  135. except RequirementParseError:
  136. logger.debug(
  137. "Cannot parse '%s' as requirements file", req, exc_info=True
  138. )
  139. else:
  140. msg += " File '{}' does not exist.".format(req)
  141. return msg
  142. class RequirementParts(object):
  143. def __init__(
  144. self,
  145. requirement, # type: Optional[Requirement]
  146. link, # type: Optional[Link]
  147. markers, # type: Optional[Marker]
  148. extras, # type: Set[str]
  149. ):
  150. self.requirement = requirement
  151. self.link = link
  152. self.markers = markers
  153. self.extras = extras
  154. def parse_req_from_editable(editable_req):
  155. # type: (str) -> RequirementParts
  156. name, url, extras_override = parse_editable(editable_req)
  157. if name is not None:
  158. try:
  159. req = Requirement(name)
  160. except InvalidRequirement:
  161. raise InstallationError("Invalid requirement: '{}'".format(name))
  162. else:
  163. req = None
  164. link = Link(url)
  165. return RequirementParts(req, link, None, extras_override)
  166. # ---- The actual constructors follow ----
  167. def install_req_from_editable(
  168. editable_req, # type: str
  169. comes_from=None, # type: Optional[Union[InstallRequirement, str]]
  170. use_pep517=None, # type: Optional[bool]
  171. isolated=False, # type: bool
  172. options=None, # type: Optional[Dict[str, Any]]
  173. constraint=False, # type: bool
  174. user_supplied=False, # type: bool
  175. ):
  176. # type: (...) -> InstallRequirement
  177. parts = parse_req_from_editable(editable_req)
  178. return InstallRequirement(
  179. parts.requirement,
  180. comes_from=comes_from,
  181. user_supplied=user_supplied,
  182. editable=True,
  183. link=parts.link,
  184. constraint=constraint,
  185. use_pep517=use_pep517,
  186. isolated=isolated,
  187. install_options=options.get("install_options", []) if options else [],
  188. global_options=options.get("global_options", []) if options else [],
  189. hash_options=options.get("hashes", {}) if options else {},
  190. extras=parts.extras,
  191. )
  192. def _looks_like_path(name):
  193. # type: (str) -> bool
  194. """Checks whether the string "looks like" a path on the filesystem.
  195. This does not check whether the target actually exists, only judge from the
  196. appearance.
  197. Returns true if any of the following conditions is true:
  198. * a path separator is found (either os.path.sep or os.path.altsep);
  199. * a dot is found (which represents the current directory).
  200. """
  201. if os.path.sep in name:
  202. return True
  203. if os.path.altsep is not None and os.path.altsep in name:
  204. return True
  205. if name.startswith("."):
  206. return True
  207. return False
  208. def _get_url_from_path(path, name):
  209. # type: (str, str) -> Optional[str]
  210. """
  211. First, it checks whether a provided path is an installable directory
  212. (e.g. it has a setup.py). If it is, returns the path.
  213. If false, check if the path is an archive file (such as a .whl).
  214. The function checks if the path is a file. If false, if the path has
  215. an @, it will treat it as a PEP 440 URL requirement and return the path.
  216. """
  217. if _looks_like_path(name) and os.path.isdir(path):
  218. if is_installable_dir(path):
  219. return path_to_url(path)
  220. raise InstallationError(
  221. "Directory {name!r} is not installable. Neither 'setup.py' "
  222. "nor 'pyproject.toml' found.".format(**locals())
  223. )
  224. if not is_archive_file(path):
  225. return None
  226. if os.path.isfile(path):
  227. return path_to_url(path)
  228. urlreq_parts = name.split('@', 1)
  229. if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]):
  230. # If the path contains '@' and the part before it does not look
  231. # like a path, try to treat it as a PEP 440 URL req instead.
  232. return None
  233. logger.warning(
  234. 'Requirement %r looks like a filename, but the '
  235. 'file does not exist',
  236. name
  237. )
  238. return path_to_url(path)
  239. def parse_req_from_line(name, line_source):
  240. # type: (str, Optional[str]) -> RequirementParts
  241. if is_url(name):
  242. marker_sep = '; '
  243. else:
  244. marker_sep = ';'
  245. if marker_sep in name:
  246. name, markers_as_string = name.split(marker_sep, 1)
  247. markers_as_string = markers_as_string.strip()
  248. if not markers_as_string:
  249. markers = None
  250. else:
  251. markers = Marker(markers_as_string)
  252. else:
  253. markers = None
  254. name = name.strip()
  255. req_as_string = None
  256. path = os.path.normpath(os.path.abspath(name))
  257. link = None
  258. extras_as_string = None
  259. if is_url(name):
  260. link = Link(name)
  261. else:
  262. p, extras_as_string = _strip_extras(path)
  263. url = _get_url_from_path(p, name)
  264. if url is not None:
  265. link = Link(url)
  266. # it's a local file, dir, or url
  267. if link:
  268. # Handle relative file URLs
  269. if link.scheme == 'file' and re.search(r'\.\./', link.url):
  270. link = Link(
  271. path_to_url(os.path.normpath(os.path.abspath(link.path))))
  272. # wheel file
  273. if link.is_wheel:
  274. wheel = Wheel(link.filename) # can raise InvalidWheelFilename
  275. req_as_string = "{wheel.name}=={wheel.version}".format(**locals())
  276. else:
  277. # set the req to the egg fragment. when it's not there, this
  278. # will become an 'unnamed' requirement
  279. req_as_string = link.egg_fragment
  280. # a requirement specifier
  281. else:
  282. req_as_string = name
  283. extras = convert_extras(extras_as_string)
  284. def with_source(text):
  285. # type: (str) -> str
  286. if not line_source:
  287. return text
  288. return '{} (from {})'.format(text, line_source)
  289. if req_as_string is not None:
  290. try:
  291. req = Requirement(req_as_string)
  292. except InvalidRequirement:
  293. if os.path.sep in req_as_string:
  294. add_msg = "It looks like a path."
  295. add_msg += deduce_helpful_msg(req_as_string)
  296. elif ('=' in req_as_string and
  297. not any(op in req_as_string for op in operators)):
  298. add_msg = "= is not a valid operator. Did you mean == ?"
  299. else:
  300. add_msg = ''
  301. msg = with_source(
  302. 'Invalid requirement: {!r}'.format(req_as_string)
  303. )
  304. if add_msg:
  305. msg += '\nHint: {}'.format(add_msg)
  306. raise InstallationError(msg)
  307. else:
  308. # Deprecate extras after specifiers: "name>=1.0[extras]"
  309. # This currently works by accident because _strip_extras() parses
  310. # any extras in the end of the string and those are saved in
  311. # RequirementParts
  312. for spec in req.specifier:
  313. spec_str = str(spec)
  314. if spec_str.endswith(']'):
  315. msg = "Extras after version '{}'.".format(spec_str)
  316. replace = "moving the extras before version specifiers"
  317. deprecated(msg, replacement=replace, gone_in="21.0")
  318. else:
  319. req = None
  320. return RequirementParts(req, link, markers, extras)
  321. def install_req_from_line(
  322. name, # type: str
  323. comes_from=None, # type: Optional[Union[str, InstallRequirement]]
  324. use_pep517=None, # type: Optional[bool]
  325. isolated=False, # type: bool
  326. options=None, # type: Optional[Dict[str, Any]]
  327. constraint=False, # type: bool
  328. line_source=None, # type: Optional[str]
  329. user_supplied=False, # type: bool
  330. ):
  331. # type: (...) -> InstallRequirement
  332. """Creates an InstallRequirement from a name, which might be a
  333. requirement, directory containing 'setup.py', filename, or URL.
  334. :param line_source: An optional string describing where the line is from,
  335. for logging purposes in case of an error.
  336. """
  337. parts = parse_req_from_line(name, line_source)
  338. return InstallRequirement(
  339. parts.requirement, comes_from, link=parts.link, markers=parts.markers,
  340. use_pep517=use_pep517, isolated=isolated,
  341. install_options=options.get("install_options", []) if options else [],
  342. global_options=options.get("global_options", []) if options else [],
  343. hash_options=options.get("hashes", {}) if options else {},
  344. constraint=constraint,
  345. extras=parts.extras,
  346. user_supplied=user_supplied,
  347. )
  348. def install_req_from_req_string(
  349. req_string, # type: str
  350. comes_from=None, # type: Optional[InstallRequirement]
  351. isolated=False, # type: bool
  352. use_pep517=None, # type: Optional[bool]
  353. user_supplied=False, # type: bool
  354. ):
  355. # type: (...) -> InstallRequirement
  356. try:
  357. req = Requirement(req_string)
  358. except InvalidRequirement:
  359. raise InstallationError("Invalid requirement: '{}'".format(req_string))
  360. domains_not_allowed = [
  361. PyPI.file_storage_domain,
  362. TestPyPI.file_storage_domain,
  363. ]
  364. if (req.url and comes_from and comes_from.link and
  365. comes_from.link.netloc in domains_not_allowed):
  366. # Explicitly disallow pypi packages that depend on external urls
  367. raise InstallationError(
  368. "Packages installed from PyPI cannot depend on packages "
  369. "which are not also hosted on PyPI.\n"
  370. "{} depends on {} ".format(comes_from.name, req)
  371. )
  372. return InstallRequirement(
  373. req,
  374. comes_from,
  375. isolated=isolated,
  376. use_pep517=use_pep517,
  377. user_supplied=user_supplied,
  378. )
  379. def install_req_from_parsed_requirement(
  380. parsed_req, # type: ParsedRequirement
  381. isolated=False, # type: bool
  382. use_pep517=None, # type: Optional[bool]
  383. user_supplied=False, # type: bool
  384. ):
  385. # type: (...) -> InstallRequirement
  386. if parsed_req.is_editable:
  387. req = install_req_from_editable(
  388. parsed_req.requirement,
  389. comes_from=parsed_req.comes_from,
  390. use_pep517=use_pep517,
  391. constraint=parsed_req.constraint,
  392. isolated=isolated,
  393. user_supplied=user_supplied,
  394. )
  395. else:
  396. req = install_req_from_line(
  397. parsed_req.requirement,
  398. comes_from=parsed_req.comes_from,
  399. use_pep517=use_pep517,
  400. isolated=isolated,
  401. options=parsed_req.options,
  402. constraint=parsed_req.constraint,
  403. line_source=parsed_req.line_source,
  404. user_supplied=user_supplied,
  405. )
  406. return req