metadata.py 38 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2012 The Python Software Foundation.
  4. # See LICENSE.txt and CONTRIBUTORS.txt.
  5. #
  6. """Implementation of the Metadata for Python packages PEPs.
  7. Supports all metadata formats (1.0, 1.1, 1.2, and 2.0 experimental).
  8. """
  9. from __future__ import unicode_literals
  10. import codecs
  11. from email import message_from_file
  12. import json
  13. import logging
  14. import re
  15. from . import DistlibException, __version__
  16. from .compat import StringIO, string_types, text_type
  17. from .markers import interpret
  18. from .util import extract_by_key, get_extras
  19. from .version import get_scheme, PEP440_VERSION_RE
  20. logger = logging.getLogger(__name__)
  21. class MetadataMissingError(DistlibException):
  22. """A required metadata is missing"""
  23. class MetadataConflictError(DistlibException):
  24. """Attempt to read or write metadata fields that are conflictual."""
  25. class MetadataUnrecognizedVersionError(DistlibException):
  26. """Unknown metadata version number."""
  27. class MetadataInvalidError(DistlibException):
  28. """A metadata value is invalid"""
  29. # public API of this module
  30. __all__ = ['Metadata', 'PKG_INFO_ENCODING', 'PKG_INFO_PREFERRED_VERSION']
  31. # Encoding used for the PKG-INFO files
  32. PKG_INFO_ENCODING = 'utf-8'
  33. # preferred version. Hopefully will be changed
  34. # to 1.2 once PEP 345 is supported everywhere
  35. PKG_INFO_PREFERRED_VERSION = '1.1'
  36. _LINE_PREFIX = re.compile('\n \|')
  37. _241_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform',
  38. 'Summary', 'Description',
  39. 'Keywords', 'Home-page', 'Author', 'Author-email',
  40. 'License')
  41. _314_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform',
  42. 'Supported-Platform', 'Summary', 'Description',
  43. 'Keywords', 'Home-page', 'Author', 'Author-email',
  44. 'License', 'Classifier', 'Download-URL', 'Obsoletes',
  45. 'Provides', 'Requires')
  46. _314_MARKERS = ('Obsoletes', 'Provides', 'Requires', 'Classifier',
  47. 'Download-URL')
  48. _345_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform',
  49. 'Supported-Platform', 'Summary', 'Description',
  50. 'Keywords', 'Home-page', 'Author', 'Author-email',
  51. 'Maintainer', 'Maintainer-email', 'License',
  52. 'Classifier', 'Download-URL', 'Obsoletes-Dist',
  53. 'Project-URL', 'Provides-Dist', 'Requires-Dist',
  54. 'Requires-Python', 'Requires-External')
  55. _345_MARKERS = ('Provides-Dist', 'Requires-Dist', 'Requires-Python',
  56. 'Obsoletes-Dist', 'Requires-External', 'Maintainer',
  57. 'Maintainer-email', 'Project-URL')
  58. _426_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform',
  59. 'Supported-Platform', 'Summary', 'Description',
  60. 'Keywords', 'Home-page', 'Author', 'Author-email',
  61. 'Maintainer', 'Maintainer-email', 'License',
  62. 'Classifier', 'Download-URL', 'Obsoletes-Dist',
  63. 'Project-URL', 'Provides-Dist', 'Requires-Dist',
  64. 'Requires-Python', 'Requires-External', 'Private-Version',
  65. 'Obsoleted-By', 'Setup-Requires-Dist', 'Extension',
  66. 'Provides-Extra')
  67. _426_MARKERS = ('Private-Version', 'Provides-Extra', 'Obsoleted-By',
  68. 'Setup-Requires-Dist', 'Extension')
  69. _ALL_FIELDS = set()
  70. _ALL_FIELDS.update(_241_FIELDS)
  71. _ALL_FIELDS.update(_314_FIELDS)
  72. _ALL_FIELDS.update(_345_FIELDS)
  73. _ALL_FIELDS.update(_426_FIELDS)
  74. EXTRA_RE = re.compile(r'''extra\s*==\s*("([^"]+)"|'([^']+)')''')
  75. def _version2fieldlist(version):
  76. if version == '1.0':
  77. return _241_FIELDS
  78. elif version == '1.1':
  79. return _314_FIELDS
  80. elif version == '1.2':
  81. return _345_FIELDS
  82. elif version == '2.0':
  83. return _426_FIELDS
  84. raise MetadataUnrecognizedVersionError(version)
  85. def _best_version(fields):
  86. """Detect the best version depending on the fields used."""
  87. def _has_marker(keys, markers):
  88. for marker in markers:
  89. if marker in keys:
  90. return True
  91. return False
  92. keys = []
  93. for key, value in fields.items():
  94. if value in ([], 'UNKNOWN', None):
  95. continue
  96. keys.append(key)
  97. possible_versions = ['1.0', '1.1', '1.2', '2.0']
  98. # first let's try to see if a field is not part of one of the version
  99. for key in keys:
  100. if key not in _241_FIELDS and '1.0' in possible_versions:
  101. possible_versions.remove('1.0')
  102. if key not in _314_FIELDS and '1.1' in possible_versions:
  103. possible_versions.remove('1.1')
  104. if key not in _345_FIELDS and '1.2' in possible_versions:
  105. possible_versions.remove('1.2')
  106. if key not in _426_FIELDS and '2.0' in possible_versions:
  107. possible_versions.remove('2.0')
  108. # possible_version contains qualified versions
  109. if len(possible_versions) == 1:
  110. return possible_versions[0] # found !
  111. elif len(possible_versions) == 0:
  112. raise MetadataConflictError('Unknown metadata set')
  113. # let's see if one unique marker is found
  114. is_1_1 = '1.1' in possible_versions and _has_marker(keys, _314_MARKERS)
  115. is_1_2 = '1.2' in possible_versions and _has_marker(keys, _345_MARKERS)
  116. is_2_0 = '2.0' in possible_versions and _has_marker(keys, _426_MARKERS)
  117. if int(is_1_1) + int(is_1_2) + int(is_2_0) > 1:
  118. raise MetadataConflictError('You used incompatible 1.1/1.2/2.0 fields')
  119. # we have the choice, 1.0, or 1.2, or 2.0
  120. # - 1.0 has a broken Summary field but works with all tools
  121. # - 1.1 is to avoid
  122. # - 1.2 fixes Summary but has little adoption
  123. # - 2.0 adds more features and is very new
  124. if not is_1_1 and not is_1_2 and not is_2_0:
  125. # we couldn't find any specific marker
  126. if PKG_INFO_PREFERRED_VERSION in possible_versions:
  127. return PKG_INFO_PREFERRED_VERSION
  128. if is_1_1:
  129. return '1.1'
  130. if is_1_2:
  131. return '1.2'
  132. return '2.0'
  133. _ATTR2FIELD = {
  134. 'metadata_version': 'Metadata-Version',
  135. 'name': 'Name',
  136. 'version': 'Version',
  137. 'platform': 'Platform',
  138. 'supported_platform': 'Supported-Platform',
  139. 'summary': 'Summary',
  140. 'description': 'Description',
  141. 'keywords': 'Keywords',
  142. 'home_page': 'Home-page',
  143. 'author': 'Author',
  144. 'author_email': 'Author-email',
  145. 'maintainer': 'Maintainer',
  146. 'maintainer_email': 'Maintainer-email',
  147. 'license': 'License',
  148. 'classifier': 'Classifier',
  149. 'download_url': 'Download-URL',
  150. 'obsoletes_dist': 'Obsoletes-Dist',
  151. 'provides_dist': 'Provides-Dist',
  152. 'requires_dist': 'Requires-Dist',
  153. 'setup_requires_dist': 'Setup-Requires-Dist',
  154. 'requires_python': 'Requires-Python',
  155. 'requires_external': 'Requires-External',
  156. 'requires': 'Requires',
  157. 'provides': 'Provides',
  158. 'obsoletes': 'Obsoletes',
  159. 'project_url': 'Project-URL',
  160. 'private_version': 'Private-Version',
  161. 'obsoleted_by': 'Obsoleted-By',
  162. 'extension': 'Extension',
  163. 'provides_extra': 'Provides-Extra',
  164. }
  165. _PREDICATE_FIELDS = ('Requires-Dist', 'Obsoletes-Dist', 'Provides-Dist')
  166. _VERSIONS_FIELDS = ('Requires-Python',)
  167. _VERSION_FIELDS = ('Version',)
  168. _LISTFIELDS = ('Platform', 'Classifier', 'Obsoletes',
  169. 'Requires', 'Provides', 'Obsoletes-Dist',
  170. 'Provides-Dist', 'Requires-Dist', 'Requires-External',
  171. 'Project-URL', 'Supported-Platform', 'Setup-Requires-Dist',
  172. 'Provides-Extra', 'Extension')
  173. _LISTTUPLEFIELDS = ('Project-URL',)
  174. _ELEMENTSFIELD = ('Keywords',)
  175. _UNICODEFIELDS = ('Author', 'Maintainer', 'Summary', 'Description')
  176. _MISSING = object()
  177. _FILESAFE = re.compile('[^A-Za-z0-9.]+')
  178. def _get_name_and_version(name, version, for_filename=False):
  179. """Return the distribution name with version.
  180. If for_filename is true, return a filename-escaped form."""
  181. if for_filename:
  182. # For both name and version any runs of non-alphanumeric or '.'
  183. # characters are replaced with a single '-'. Additionally any
  184. # spaces in the version string become '.'
  185. name = _FILESAFE.sub('-', name)
  186. version = _FILESAFE.sub('-', version.replace(' ', '.'))
  187. return '%s-%s' % (name, version)
  188. class LegacyMetadata(object):
  189. """The legacy metadata of a release.
  190. Supports versions 1.0, 1.1 and 1.2 (auto-detected). You can
  191. instantiate the class with one of these arguments (or none):
  192. - *path*, the path to a metadata file
  193. - *fileobj* give a file-like object with metadata as content
  194. - *mapping* is a dict-like object
  195. - *scheme* is a version scheme name
  196. """
  197. # TODO document the mapping API and UNKNOWN default key
  198. def __init__(self, path=None, fileobj=None, mapping=None,
  199. scheme='default'):
  200. if [path, fileobj, mapping].count(None) < 2:
  201. raise TypeError('path, fileobj and mapping are exclusive')
  202. self._fields = {}
  203. self.requires_files = []
  204. self._dependencies = None
  205. self.scheme = scheme
  206. if path is not None:
  207. self.read(path)
  208. elif fileobj is not None:
  209. self.read_file(fileobj)
  210. elif mapping is not None:
  211. self.update(mapping)
  212. self.set_metadata_version()
  213. def set_metadata_version(self):
  214. self._fields['Metadata-Version'] = _best_version(self._fields)
  215. def _write_field(self, fileobj, name, value):
  216. fileobj.write('%s: %s\n' % (name, value))
  217. def __getitem__(self, name):
  218. return self.get(name)
  219. def __setitem__(self, name, value):
  220. return self.set(name, value)
  221. def __delitem__(self, name):
  222. field_name = self._convert_name(name)
  223. try:
  224. del self._fields[field_name]
  225. except KeyError:
  226. raise KeyError(name)
  227. def __contains__(self, name):
  228. return (name in self._fields or
  229. self._convert_name(name) in self._fields)
  230. def _convert_name(self, name):
  231. if name in _ALL_FIELDS:
  232. return name
  233. name = name.replace('-', '_').lower()
  234. return _ATTR2FIELD.get(name, name)
  235. def _default_value(self, name):
  236. if name in _LISTFIELDS or name in _ELEMENTSFIELD:
  237. return []
  238. return 'UNKNOWN'
  239. def _remove_line_prefix(self, value):
  240. return _LINE_PREFIX.sub('\n', value)
  241. def __getattr__(self, name):
  242. if name in _ATTR2FIELD:
  243. return self[name]
  244. raise AttributeError(name)
  245. #
  246. # Public API
  247. #
  248. # dependencies = property(_get_dependencies, _set_dependencies)
  249. def get_fullname(self, filesafe=False):
  250. """Return the distribution name with version.
  251. If filesafe is true, return a filename-escaped form."""
  252. return _get_name_and_version(self['Name'], self['Version'], filesafe)
  253. def is_field(self, name):
  254. """return True if name is a valid metadata key"""
  255. name = self._convert_name(name)
  256. return name in _ALL_FIELDS
  257. def is_multi_field(self, name):
  258. name = self._convert_name(name)
  259. return name in _LISTFIELDS
  260. def read(self, filepath):
  261. """Read the metadata values from a file path."""
  262. fp = codecs.open(filepath, 'r', encoding='utf-8')
  263. try:
  264. self.read_file(fp)
  265. finally:
  266. fp.close()
  267. def read_file(self, fileob):
  268. """Read the metadata values from a file object."""
  269. msg = message_from_file(fileob)
  270. self._fields['Metadata-Version'] = msg['metadata-version']
  271. # When reading, get all the fields we can
  272. for field in _ALL_FIELDS:
  273. if field not in msg:
  274. continue
  275. if field in _LISTFIELDS:
  276. # we can have multiple lines
  277. values = msg.get_all(field)
  278. if field in _LISTTUPLEFIELDS and values is not None:
  279. values = [tuple(value.split(',')) for value in values]
  280. self.set(field, values)
  281. else:
  282. # single line
  283. value = msg[field]
  284. if value is not None and value != 'UNKNOWN':
  285. self.set(field, value)
  286. self.set_metadata_version()
  287. def write(self, filepath, skip_unknown=False):
  288. """Write the metadata fields to filepath."""
  289. fp = codecs.open(filepath, 'w', encoding='utf-8')
  290. try:
  291. self.write_file(fp, skip_unknown)
  292. finally:
  293. fp.close()
  294. def write_file(self, fileobject, skip_unknown=False):
  295. """Write the PKG-INFO format data to a file object."""
  296. self.set_metadata_version()
  297. for field in _version2fieldlist(self['Metadata-Version']):
  298. values = self.get(field)
  299. if skip_unknown and values in ('UNKNOWN', [], ['UNKNOWN']):
  300. continue
  301. if field in _ELEMENTSFIELD:
  302. self._write_field(fileobject, field, ','.join(values))
  303. continue
  304. if field not in _LISTFIELDS:
  305. if field == 'Description':
  306. values = values.replace('\n', '\n |')
  307. values = [values]
  308. if field in _LISTTUPLEFIELDS:
  309. values = [','.join(value) for value in values]
  310. for value in values:
  311. self._write_field(fileobject, field, value)
  312. def update(self, other=None, **kwargs):
  313. """Set metadata values from the given iterable `other` and kwargs.
  314. Behavior is like `dict.update`: If `other` has a ``keys`` method,
  315. they are looped over and ``self[key]`` is assigned ``other[key]``.
  316. Else, ``other`` is an iterable of ``(key, value)`` iterables.
  317. Keys that don't match a metadata field or that have an empty value are
  318. dropped.
  319. """
  320. def _set(key, value):
  321. if key in _ATTR2FIELD and value:
  322. self.set(self._convert_name(key), value)
  323. if not other:
  324. # other is None or empty container
  325. pass
  326. elif hasattr(other, 'keys'):
  327. for k in other.keys():
  328. _set(k, other[k])
  329. else:
  330. for k, v in other:
  331. _set(k, v)
  332. if kwargs:
  333. for k, v in kwargs.items():
  334. _set(k, v)
  335. def set(self, name, value):
  336. """Control then set a metadata field."""
  337. name = self._convert_name(name)
  338. if ((name in _ELEMENTSFIELD or name == 'Platform') and
  339. not isinstance(value, (list, tuple))):
  340. if isinstance(value, string_types):
  341. value = [v.strip() for v in value.split(',')]
  342. else:
  343. value = []
  344. elif (name in _LISTFIELDS and
  345. not isinstance(value, (list, tuple))):
  346. if isinstance(value, string_types):
  347. value = [value]
  348. else:
  349. value = []
  350. if logger.isEnabledFor(logging.WARNING):
  351. project_name = self['Name']
  352. scheme = get_scheme(self.scheme)
  353. if name in _PREDICATE_FIELDS and value is not None:
  354. for v in value:
  355. # check that the values are valid
  356. if not scheme.is_valid_matcher(v.split(';')[0]):
  357. logger.warning(
  358. '%r: %r is not valid (field %r)',
  359. project_name, v, name)
  360. # FIXME this rejects UNKNOWN, is that right?
  361. elif name in _VERSIONS_FIELDS and value is not None:
  362. if not scheme.is_valid_constraint_list(value):
  363. logger.warning('%r: %r is not a valid version (field %r)',
  364. project_name, value, name)
  365. elif name in _VERSION_FIELDS and value is not None:
  366. if not scheme.is_valid_version(value):
  367. logger.warning('%r: %r is not a valid version (field %r)',
  368. project_name, value, name)
  369. if name in _UNICODEFIELDS:
  370. if name == 'Description':
  371. value = self._remove_line_prefix(value)
  372. self._fields[name] = value
  373. def get(self, name, default=_MISSING):
  374. """Get a metadata field."""
  375. name = self._convert_name(name)
  376. if name not in self._fields:
  377. if default is _MISSING:
  378. default = self._default_value(name)
  379. return default
  380. if name in _UNICODEFIELDS:
  381. value = self._fields[name]
  382. return value
  383. elif name in _LISTFIELDS:
  384. value = self._fields[name]
  385. if value is None:
  386. return []
  387. res = []
  388. for val in value:
  389. if name not in _LISTTUPLEFIELDS:
  390. res.append(val)
  391. else:
  392. # That's for Project-URL
  393. res.append((val[0], val[1]))
  394. return res
  395. elif name in _ELEMENTSFIELD:
  396. value = self._fields[name]
  397. if isinstance(value, string_types):
  398. return value.split(',')
  399. return self._fields[name]
  400. def check(self, strict=False):
  401. """Check if the metadata is compliant. If strict is True then raise if
  402. no Name or Version are provided"""
  403. self.set_metadata_version()
  404. # XXX should check the versions (if the file was loaded)
  405. missing, warnings = [], []
  406. for attr in ('Name', 'Version'): # required by PEP 345
  407. if attr not in self:
  408. missing.append(attr)
  409. if strict and missing != []:
  410. msg = 'missing required metadata: %s' % ', '.join(missing)
  411. raise MetadataMissingError(msg)
  412. for attr in ('Home-page', 'Author'):
  413. if attr not in self:
  414. missing.append(attr)
  415. # checking metadata 1.2 (XXX needs to check 1.1, 1.0)
  416. if self['Metadata-Version'] != '1.2':
  417. return missing, warnings
  418. scheme = get_scheme(self.scheme)
  419. def are_valid_constraints(value):
  420. for v in value:
  421. if not scheme.is_valid_matcher(v.split(';')[0]):
  422. return False
  423. return True
  424. for fields, controller in ((_PREDICATE_FIELDS, are_valid_constraints),
  425. (_VERSIONS_FIELDS,
  426. scheme.is_valid_constraint_list),
  427. (_VERSION_FIELDS,
  428. scheme.is_valid_version)):
  429. for field in fields:
  430. value = self.get(field, None)
  431. if value is not None and not controller(value):
  432. warnings.append('Wrong value for %r: %s' % (field, value))
  433. return missing, warnings
  434. def todict(self, skip_missing=False):
  435. """Return fields as a dict.
  436. Field names will be converted to use the underscore-lowercase style
  437. instead of hyphen-mixed case (i.e. home_page instead of Home-page).
  438. """
  439. self.set_metadata_version()
  440. mapping_1_0 = (
  441. ('metadata_version', 'Metadata-Version'),
  442. ('name', 'Name'),
  443. ('version', 'Version'),
  444. ('summary', 'Summary'),
  445. ('home_page', 'Home-page'),
  446. ('author', 'Author'),
  447. ('author_email', 'Author-email'),
  448. ('license', 'License'),
  449. ('description', 'Description'),
  450. ('keywords', 'Keywords'),
  451. ('platform', 'Platform'),
  452. ('classifier', 'Classifier'),
  453. ('download_url', 'Download-URL'),
  454. )
  455. data = {}
  456. for key, field_name in mapping_1_0:
  457. if not skip_missing or field_name in self._fields:
  458. data[key] = self[field_name]
  459. if self['Metadata-Version'] == '1.2':
  460. mapping_1_2 = (
  461. ('requires_dist', 'Requires-Dist'),
  462. ('requires_python', 'Requires-Python'),
  463. ('requires_external', 'Requires-External'),
  464. ('provides_dist', 'Provides-Dist'),
  465. ('obsoletes_dist', 'Obsoletes-Dist'),
  466. ('project_url', 'Project-URL'),
  467. ('maintainer', 'Maintainer'),
  468. ('maintainer_email', 'Maintainer-email'),
  469. )
  470. for key, field_name in mapping_1_2:
  471. if not skip_missing or field_name in self._fields:
  472. if key != 'project_url':
  473. data[key] = self[field_name]
  474. else:
  475. data[key] = [','.join(u) for u in self[field_name]]
  476. elif self['Metadata-Version'] == '1.1':
  477. mapping_1_1 = (
  478. ('provides', 'Provides'),
  479. ('requires', 'Requires'),
  480. ('obsoletes', 'Obsoletes'),
  481. )
  482. for key, field_name in mapping_1_1:
  483. if not skip_missing or field_name in self._fields:
  484. data[key] = self[field_name]
  485. return data
  486. def add_requirements(self, requirements):
  487. if self['Metadata-Version'] == '1.1':
  488. # we can't have 1.1 metadata *and* Setuptools requires
  489. for field in ('Obsoletes', 'Requires', 'Provides'):
  490. if field in self:
  491. del self[field]
  492. self['Requires-Dist'] += requirements
  493. # Mapping API
  494. # TODO could add iter* variants
  495. def keys(self):
  496. return list(_version2fieldlist(self['Metadata-Version']))
  497. def __iter__(self):
  498. for key in self.keys():
  499. yield key
  500. def values(self):
  501. return [self[key] for key in self.keys()]
  502. def items(self):
  503. return [(key, self[key]) for key in self.keys()]
  504. def __repr__(self):
  505. return '<%s %s %s>' % (self.__class__.__name__, self.name,
  506. self.version)
  507. METADATA_FILENAME = 'pydist.json'
  508. class Metadata(object):
  509. """
  510. The metadata of a release. This implementation uses 2.0 (JSON)
  511. metadata where possible. If not possible, it wraps a LegacyMetadata
  512. instance which handles the key-value metadata format.
  513. """
  514. METADATA_VERSION_MATCHER = re.compile('^\d+(\.\d+)*$')
  515. NAME_MATCHER = re.compile('^[0-9A-Z]([0-9A-Z_.-]*[0-9A-Z])?$', re.I)
  516. VERSION_MATCHER = PEP440_VERSION_RE
  517. SUMMARY_MATCHER = re.compile('.{1,2047}')
  518. METADATA_VERSION = '2.0'
  519. GENERATOR = 'distlib (%s)' % __version__
  520. MANDATORY_KEYS = {
  521. 'name': (),
  522. 'version': (),
  523. 'summary': ('legacy',),
  524. }
  525. INDEX_KEYS = ('name version license summary description author '
  526. 'author_email keywords platform home_page classifiers '
  527. 'download_url')
  528. DEPENDENCY_KEYS = ('extras run_requires test_requires build_requires '
  529. 'dev_requires provides meta_requires obsoleted_by '
  530. 'supports_environments')
  531. SYNTAX_VALIDATORS = {
  532. 'metadata_version': (METADATA_VERSION_MATCHER, ()),
  533. 'name': (NAME_MATCHER, ('legacy',)),
  534. 'version': (VERSION_MATCHER, ('legacy',)),
  535. 'summary': (SUMMARY_MATCHER, ('legacy',)),
  536. }
  537. __slots__ = ('_legacy', '_data', 'scheme')
  538. def __init__(self, path=None, fileobj=None, mapping=None,
  539. scheme='default'):
  540. if [path, fileobj, mapping].count(None) < 2:
  541. raise TypeError('path, fileobj and mapping are exclusive')
  542. self._legacy = None
  543. self._data = None
  544. self.scheme = scheme
  545. #import pdb; pdb.set_trace()
  546. if mapping is not None:
  547. try:
  548. self._validate_mapping(mapping, scheme)
  549. self._data = mapping
  550. except MetadataUnrecognizedVersionError:
  551. self._legacy = LegacyMetadata(mapping=mapping, scheme=scheme)
  552. self.validate()
  553. else:
  554. data = None
  555. if path:
  556. with open(path, 'rb') as f:
  557. data = f.read()
  558. elif fileobj:
  559. data = fileobj.read()
  560. if data is None:
  561. # Initialised with no args - to be added
  562. self._data = {
  563. 'metadata_version': self.METADATA_VERSION,
  564. 'generator': self.GENERATOR,
  565. }
  566. else:
  567. if not isinstance(data, text_type):
  568. data = data.decode('utf-8')
  569. try:
  570. self._data = json.loads(data)
  571. self._validate_mapping(self._data, scheme)
  572. except ValueError:
  573. # Note: MetadataUnrecognizedVersionError does not
  574. # inherit from ValueError (it's a DistlibException,
  575. # which should not inherit from ValueError).
  576. # The ValueError comes from the json.load - if that
  577. # succeeds and we get a validation error, we want
  578. # that to propagate
  579. self._legacy = LegacyMetadata(fileobj=StringIO(data),
  580. scheme=scheme)
  581. self.validate()
  582. common_keys = set(('name', 'version', 'license', 'keywords', 'summary'))
  583. none_list = (None, list)
  584. none_dict = (None, dict)
  585. mapped_keys = {
  586. 'run_requires': ('Requires-Dist', list),
  587. 'build_requires': ('Setup-Requires-Dist', list),
  588. 'dev_requires': none_list,
  589. 'test_requires': none_list,
  590. 'meta_requires': none_list,
  591. 'extras': ('Provides-Extra', list),
  592. 'modules': none_list,
  593. 'namespaces': none_list,
  594. 'exports': none_dict,
  595. 'commands': none_dict,
  596. 'classifiers': ('Classifier', list),
  597. 'source_url': ('Download-URL', None),
  598. 'metadata_version': ('Metadata-Version', None),
  599. }
  600. del none_list, none_dict
  601. def __getattribute__(self, key):
  602. common = object.__getattribute__(self, 'common_keys')
  603. mapped = object.__getattribute__(self, 'mapped_keys')
  604. if key in mapped:
  605. lk, maker = mapped[key]
  606. if self._legacy:
  607. if lk is None:
  608. result = None if maker is None else maker()
  609. else:
  610. result = self._legacy.get(lk)
  611. else:
  612. value = None if maker is None else maker()
  613. if key not in ('commands', 'exports', 'modules', 'namespaces',
  614. 'classifiers'):
  615. result = self._data.get(key, value)
  616. else:
  617. # special cases for PEP 459
  618. sentinel = object()
  619. result = sentinel
  620. d = self._data.get('extensions')
  621. if d:
  622. if key == 'commands':
  623. result = d.get('python.commands', value)
  624. elif key == 'classifiers':
  625. d = d.get('python.details')
  626. if d:
  627. result = d.get(key, value)
  628. else:
  629. d = d.get('python.exports')
  630. if d:
  631. result = d.get(key, value)
  632. if result is sentinel:
  633. result = value
  634. elif key not in common:
  635. result = object.__getattribute__(self, key)
  636. elif self._legacy:
  637. result = self._legacy.get(key)
  638. else:
  639. result = self._data.get(key)
  640. return result
  641. def _validate_value(self, key, value, scheme=None):
  642. if key in self.SYNTAX_VALIDATORS:
  643. pattern, exclusions = self.SYNTAX_VALIDATORS[key]
  644. if (scheme or self.scheme) not in exclusions:
  645. m = pattern.match(value)
  646. if not m:
  647. raise MetadataInvalidError('%r is an invalid value for '
  648. 'the %r property' % (value,
  649. key))
  650. def __setattr__(self, key, value):
  651. self._validate_value(key, value)
  652. common = object.__getattribute__(self, 'common_keys')
  653. mapped = object.__getattribute__(self, 'mapped_keys')
  654. if key in mapped:
  655. lk, _ = mapped[key]
  656. if self._legacy:
  657. if lk is None:
  658. raise NotImplementedError
  659. self._legacy[lk] = value
  660. elif key not in ('commands', 'exports', 'modules', 'namespaces',
  661. 'classifiers'):
  662. self._data[key] = value
  663. else:
  664. # special cases for PEP 459
  665. d = self._data.setdefault('extensions', {})
  666. if key == 'commands':
  667. d['python.commands'] = value
  668. elif key == 'classifiers':
  669. d = d.setdefault('python.details', {})
  670. d[key] = value
  671. else:
  672. d = d.setdefault('python.exports', {})
  673. d[key] = value
  674. elif key not in common:
  675. object.__setattr__(self, key, value)
  676. else:
  677. if key == 'keywords':
  678. if isinstance(value, string_types):
  679. value = value.strip()
  680. if value:
  681. value = value.split()
  682. else:
  683. value = []
  684. if self._legacy:
  685. self._legacy[key] = value
  686. else:
  687. self._data[key] = value
  688. @property
  689. def name_and_version(self):
  690. return _get_name_and_version(self.name, self.version, True)
  691. @property
  692. def provides(self):
  693. if self._legacy:
  694. result = self._legacy['Provides-Dist']
  695. else:
  696. result = self._data.setdefault('provides', [])
  697. s = '%s (%s)' % (self.name, self.version)
  698. if s not in result:
  699. result.append(s)
  700. return result
  701. @provides.setter
  702. def provides(self, value):
  703. if self._legacy:
  704. self._legacy['Provides-Dist'] = value
  705. else:
  706. self._data['provides'] = value
  707. def get_requirements(self, reqts, extras=None, env=None):
  708. """
  709. Base method to get dependencies, given a set of extras
  710. to satisfy and an optional environment context.
  711. :param reqts: A list of sometimes-wanted dependencies,
  712. perhaps dependent on extras and environment.
  713. :param extras: A list of optional components being requested.
  714. :param env: An optional environment for marker evaluation.
  715. """
  716. if self._legacy:
  717. result = reqts
  718. else:
  719. result = []
  720. extras = get_extras(extras or [], self.extras)
  721. for d in reqts:
  722. if 'extra' not in d and 'environment' not in d:
  723. # unconditional
  724. include = True
  725. else:
  726. if 'extra' not in d:
  727. # Not extra-dependent - only environment-dependent
  728. include = True
  729. else:
  730. include = d.get('extra') in extras
  731. if include:
  732. # Not excluded because of extras, check environment
  733. marker = d.get('environment')
  734. if marker:
  735. include = interpret(marker, env)
  736. if include:
  737. result.extend(d['requires'])
  738. for key in ('build', 'dev', 'test'):
  739. e = ':%s:' % key
  740. if e in extras:
  741. extras.remove(e)
  742. # A recursive call, but it should terminate since 'test'
  743. # has been removed from the extras
  744. reqts = self._data.get('%s_requires' % key, [])
  745. result.extend(self.get_requirements(reqts, extras=extras,
  746. env=env))
  747. return result
  748. @property
  749. def dictionary(self):
  750. if self._legacy:
  751. return self._from_legacy()
  752. return self._data
  753. @property
  754. def dependencies(self):
  755. if self._legacy:
  756. raise NotImplementedError
  757. else:
  758. return extract_by_key(self._data, self.DEPENDENCY_KEYS)
  759. @dependencies.setter
  760. def dependencies(self, value):
  761. if self._legacy:
  762. raise NotImplementedError
  763. else:
  764. self._data.update(value)
  765. def _validate_mapping(self, mapping, scheme):
  766. if mapping.get('metadata_version') != self.METADATA_VERSION:
  767. raise MetadataUnrecognizedVersionError()
  768. missing = []
  769. for key, exclusions in self.MANDATORY_KEYS.items():
  770. if key not in mapping:
  771. if scheme not in exclusions:
  772. missing.append(key)
  773. if missing:
  774. msg = 'Missing metadata items: %s' % ', '.join(missing)
  775. raise MetadataMissingError(msg)
  776. for k, v in mapping.items():
  777. self._validate_value(k, v, scheme)
  778. def validate(self):
  779. if self._legacy:
  780. missing, warnings = self._legacy.check(True)
  781. if missing or warnings:
  782. logger.warning('Metadata: missing: %s, warnings: %s',
  783. missing, warnings)
  784. else:
  785. self._validate_mapping(self._data, self.scheme)
  786. def todict(self):
  787. if self._legacy:
  788. return self._legacy.todict(True)
  789. else:
  790. result = extract_by_key(self._data, self.INDEX_KEYS)
  791. return result
  792. def _from_legacy(self):
  793. assert self._legacy and not self._data
  794. result = {
  795. 'metadata_version': self.METADATA_VERSION,
  796. 'generator': self.GENERATOR,
  797. }
  798. lmd = self._legacy.todict(True) # skip missing ones
  799. for k in ('name', 'version', 'license', 'summary', 'description',
  800. 'classifier'):
  801. if k in lmd:
  802. if k == 'classifier':
  803. nk = 'classifiers'
  804. else:
  805. nk = k
  806. result[nk] = lmd[k]
  807. kw = lmd.get('Keywords', [])
  808. if kw == ['']:
  809. kw = []
  810. result['keywords'] = kw
  811. keys = (('requires_dist', 'run_requires'),
  812. ('setup_requires_dist', 'build_requires'))
  813. for ok, nk in keys:
  814. if ok in lmd and lmd[ok]:
  815. result[nk] = [{'requires': lmd[ok]}]
  816. result['provides'] = self.provides
  817. author = {}
  818. maintainer = {}
  819. return result
  820. LEGACY_MAPPING = {
  821. 'name': 'Name',
  822. 'version': 'Version',
  823. 'license': 'License',
  824. 'summary': 'Summary',
  825. 'description': 'Description',
  826. 'classifiers': 'Classifier',
  827. }
  828. def _to_legacy(self):
  829. def process_entries(entries):
  830. reqts = set()
  831. for e in entries:
  832. extra = e.get('extra')
  833. env = e.get('environment')
  834. rlist = e['requires']
  835. for r in rlist:
  836. if not env and not extra:
  837. reqts.add(r)
  838. else:
  839. marker = ''
  840. if extra:
  841. marker = 'extra == "%s"' % extra
  842. if env:
  843. if marker:
  844. marker = '(%s) and %s' % (env, marker)
  845. else:
  846. marker = env
  847. reqts.add(';'.join((r, marker)))
  848. return reqts
  849. assert self._data and not self._legacy
  850. result = LegacyMetadata()
  851. nmd = self._data
  852. for nk, ok in self.LEGACY_MAPPING.items():
  853. if nk in nmd:
  854. result[ok] = nmd[nk]
  855. r1 = process_entries(self.run_requires + self.meta_requires)
  856. r2 = process_entries(self.build_requires + self.dev_requires)
  857. if self.extras:
  858. result['Provides-Extra'] = sorted(self.extras)
  859. result['Requires-Dist'] = sorted(r1)
  860. result['Setup-Requires-Dist'] = sorted(r2)
  861. # TODO: other fields such as contacts
  862. return result
  863. def write(self, path=None, fileobj=None, legacy=False, skip_unknown=True):
  864. if [path, fileobj].count(None) != 1:
  865. raise ValueError('Exactly one of path and fileobj is needed')
  866. self.validate()
  867. if legacy:
  868. if self._legacy:
  869. legacy_md = self._legacy
  870. else:
  871. legacy_md = self._to_legacy()
  872. if path:
  873. legacy_md.write(path, skip_unknown=skip_unknown)
  874. else:
  875. legacy_md.write_file(fileobj, skip_unknown=skip_unknown)
  876. else:
  877. if self._legacy:
  878. d = self._from_legacy()
  879. else:
  880. d = self._data
  881. if fileobj:
  882. json.dump(d, fileobj, ensure_ascii=True, indent=2,
  883. sort_keys=True)
  884. else:
  885. with codecs.open(path, 'w', 'utf-8') as f:
  886. json.dump(d, f, ensure_ascii=True, indent=2,
  887. sort_keys=True)
  888. def add_requirements(self, requirements):
  889. if self._legacy:
  890. self._legacy.add_requirements(requirements)
  891. else:
  892. run_requires = self._data.setdefault('run_requires', [])
  893. always = None
  894. for entry in run_requires:
  895. if 'environment' not in entry and 'extra' not in entry:
  896. always = entry
  897. break
  898. if always is None:
  899. always = { 'requires': requirements }
  900. run_requires.insert(0, always)
  901. else:
  902. rset = set(always['requires']) | set(requirements)
  903. always['requires'] = sorted(rset)
  904. def __repr__(self):
  905. name = self.name or '(no name)'
  906. version = self.version or 'no version'
  907. return '<%s %s %s (%s)>' % (self.__class__.__name__,
  908. self.metadata_version, name, version)