diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 7eb37e6a73f..d712c95ecbe 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -337,9 +337,58 @@ the project path. This is one advantage over just using ``setup.py develop``, which creates the "egg-info" directly relative the current working directory. +Build System Interface +++++++++++++++++++++++ + +In order for pip to install a package from source, pip must recognise the build +system. Today only one build system is recognised, with two variants. + +If there is a ``setup.cfg`` with any of ``[extras]``, ``install-requires``, +``setup-requires``, or ``requires-dist`` present then a declarative setuptools +package is detected. + +Otherwise there is a ``setup.py`` then non-declarative setuptools is assumed. + +Declarative setuptools +~~~~~~~~~~~~~~~~~~~~~~ + +``setup.py`` must implement the following commands:: + + setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] + +With declarative dependencies, easy_install - the ``setup_requires`` keyword +is never triggered, as pip can take care of installing the requirements before +``setup.py`` is invoked. + +``setup.cfg`` must contain the package name, and one or more of setup requires, +install requires or extra requires:: + + [metadata] + name = Example + setup-requires = + somedep + install-requires = + runtimedep + [extras] + one = + anotherdep + tests = + mytestdep + +For compatibility with ``d2to1`` ``requires-dist`` is accepted as an alias for +``install-requires``, though if both are supplied an error will occur. + +Non-declarative setuptools +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``setup.py`` must implement the following commands:: + + setup.py egg_info [--egg-base XXX] + setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] -Controlling setup_requires -++++++++++++++++++++++++++ +The ``egg_info`` command should create egg metadata for the package, as +described in the setuptools documentation at +http://pythonhosted.org/setuptools/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags Setuptools offers the ``setup_requires`` `setup() keyword `_ @@ -370,19 +419,8 @@ To have the dependency located from a local directory and not crawl PyPI, add th allow_hosts = '' find_links = file:///path/to/local/archives - -Build System Interface -++++++++++++++++++++++ - -In order for pip to install a package from source, ``setup.py`` must implement -the following commands:: - - setup.py egg_info [--egg-base XXX] - setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX] - -The ``egg_info`` command should create egg metadata for the package, as -described in the setuptools documentation at -http://pythonhosted.org/setuptools/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags +Common setuptools behaviour +~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``install`` command should implement the complete process of installing the package to the target directory XXX. @@ -404,13 +442,14 @@ Investigate in more detail when this command is required). No other build system commands are invoked by the ``pip install`` command. +Wheels +~~~~~~ + Installing a package from a wheel does not invoke the build system at all. .. _PyPI: http://pypi.python.org/pypi/ .. _setuptools extras: http://packages.python.org/setuptools/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies - - .. _`pip install Options`: Options diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 65868eb7bd3..e293a1a8d0e 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -7,6 +7,7 @@ from pip._vendor import pkg_resources from pip._vendor import requests +from pip._vendor.six.moves import configparser from pip.download import (url_to_path, unpack_url) from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled, @@ -80,6 +81,16 @@ def prep_for_dist(self): raise NotImplementedError(self.dist) +def _sdist_or_static(req_to_install): + result = IsStaticMetadata(req_to_install) + try: + result.dist(None) + return result + except (configparser.NoSectionError, configparser.NoOptionError, + BadStaticSetupCfg): + return IsSDist(req_to_install) + + def make_abstract_dist(req_to_install): """Factory to make an abstract dist object. @@ -89,11 +100,11 @@ def make_abstract_dist(req_to_install): :return: A concrete DistAbstraction. """ if req_to_install.editable: - return IsSDist(req_to_install) + return _sdist_or_static(req_to_install) elif req_to_install.link and req_to_install.link.is_wheel: return IsWheel(req_to_install) else: - return IsSDist(req_to_install) + return _sdist_or_static(req_to_install) class IsWheel(DistAbstraction): @@ -123,6 +134,98 @@ def prep_for_dist(self): self.req_to_install.assert_source_matches_version() +class SetupCfgDistribution(pkg_resources.Distribution): + """A Distribution object that consults setup.cfg.""" + + def __init__(self, cfg, location=None): + self._cfg = cfg + project_name = cfg.get('metadata', 'name') + super(SetupCfgDistribution, self).__init__( + location=location, project_name=project_name) + + @property + def _dep_map(self): + # Distribution._dep_map is not generic - its expressed in terms of the + # requires.txt format from within an egg info directory, and the + # metadata interfae within Distribution looks for files-and-lines. + # To provide metadata from a ConfigParser we could either marshall + # the data to a VFS, or we can reimplement this property which is + # the primary worker to obtain requirements. + try: + return self.__dep_map + except AttributeError: + # requires + requires = self._option('install-requires', 'requires-dist') + extras = ( + self._cfg.has_section('extras') + and self._cfg.items('extras')) or [] + dm = {} + dm.setdefault(None, []).extend( + pkg_resources.parse_requirements(requires)) + for extra, extra_reqs in extras: + dm.setdefault(extra, []).extend( + pkg_resources.parse_requirements(extra_reqs)) + self.__dep_map = dm + return dm + + def setup_requires(self): + try: + setup_requires = self._cfg.get('metadata', 'setup-requires') + except configparser.NoOptionError: + return [] + return list(pkg_resources.parse_requirements(setup_requires)) + + def _option(self, option1, option2): + try: + result = self._cfg.get('metadata', option1) + if self._cfg.has_option('metadata', option2): + raise BadStaticSetupCfg('both %s and %s' % (option1, option2)) + return result + except configparser.NoOptionError: + try: + return self._cfg.get('metadata', option2) + except configparser.NoOptionError: + return [] + + +class BadStaticSetupCfg(Exception): + pass + + +def _validate_static(dist): + has_deps = False + for deps in dist._dep_map.values(): + if deps: + has_deps = True + break + if not (dist.project_name and (has_deps or dist.setup_requires())): + raise BadStaticSetupCfg() + + +class IsStaticMetadata(DistAbstraction): + """A static setup.cfg based source tree. + + Must have a name and requires|setup_requires in setup.cfg to be usable. + We look for either requires or setup-requires to handle both the case where + a setup.py has setup-requires but no runtime requirements are declared, and + the case where folk have declaratively expressed their dist requirements + without needing a setup-requires. + """ + def dist(self, finder): + cfg = configparser.SafeConfigParser() + setup_cfg = os.path.join(self.req_to_install.source_dir, 'setup.cfg') + cfg.read(setup_cfg) + dist = SetupCfgDistribution(cfg=cfg, location=setup_cfg) + _validate_static(dist) + return dist + + def prep_for_dist(self): + self._dist = self.dist(None) + self.req_to_install.req = pkg_resources.Requirement.parse( + self._dist.project_name) + self.req_to_install._correct_build_location() + + class Installed(DistAbstraction): def dist(self, finder): @@ -526,6 +629,15 @@ def add_req(subreq): self.add_requirement(req_to_install) if not self.ignore_dependencies: + if getattr(dist, 'setup_requires', None): + setup_requires = dist.setup_requires() + if setup_requires: + logger.debug( + "Installing setup_requires: %r", + ','.join(r.project_name for r in setup_requires)) + for subreq in setup_requires: + add_req(subreq) + if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", diff --git a/tests/data/packages/README.txt b/tests/data/packages/README.txt index 70f1eb08891..b2f574210e0 100644 --- a/tests/data/packages/README.txt +++ b/tests/data/packages/README.txt @@ -71,6 +71,39 @@ priority-* ---------- used for testing wheel priority over sdists +SetupRequires +------------- + +has a setup.cfg declaring a setup-requires on upper, and a setup.py that will +fail to import if upper is not installed. + +SetupRequires-0.0.1.tar.gz +-------------------------- + +SetupRequires sdisted, for testing transitive setup_requires. + +SetupRequires2 +-------------- + +has a setup.cfg declaring a dist-requires on setuprequires. Covers both +setup-requires in depended-on packages, and setup.cfg with only requires-dist +expressed. +Also in setup.cfg declares two extras - a and b, a which brings in simple +and b which brings in simple2, for testing extras from setup.cfg. + +SetupRequires2-0.0.1.tar.gz +--------------------------- + +SetupRequires2 sdisted, for testing declarative extras. + +SetupRequires3 +-------------- + +requires SetupRequires2[a,b], as using extras for local paths is currently +broken (issue 1236). Ideally SetupRequires3 would have the extras itself +and no requires-dist (to test declarative extras as sole requirements). + + simple[2]-[123].0.tar.gz ------------------------ contains "simple[2]" package; good for basic testing and version logic. diff --git a/tests/data/packages/SetupRequires-0.0.1.tar.gz b/tests/data/packages/SetupRequires-0.0.1.tar.gz new file mode 100644 index 00000000000..037d49ca321 Binary files /dev/null and b/tests/data/packages/SetupRequires-0.0.1.tar.gz differ diff --git a/tests/data/packages/SetupRequires/setup.cfg b/tests/data/packages/SetupRequires/setup.cfg new file mode 100644 index 00000000000..17b64110715 --- /dev/null +++ b/tests/data/packages/SetupRequires/setup.cfg @@ -0,0 +1,4 @@ +[metadata] +name = SetupRequires +requires-setup = + upper diff --git a/tests/data/packages/SetupRequires/setup.py b/tests/data/packages/SetupRequires/setup.py new file mode 100644 index 00000000000..a6a6cdecaf7 --- /dev/null +++ b/tests/data/packages/SetupRequires/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup +import upper + +setup( + name='SetupRequires', + version='0.0.1', + packages=['setuprequires'], +) diff --git a/tests/data/packages/SetupRequires/setuprequires/__init__.py b/tests/data/packages/SetupRequires/setuprequires/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/packages/SetupRequires2-0.0.1.tar.gz b/tests/data/packages/SetupRequires2-0.0.1.tar.gz new file mode 100644 index 00000000000..dca12125862 Binary files /dev/null and b/tests/data/packages/SetupRequires2-0.0.1.tar.gz differ diff --git a/tests/data/packages/SetupRequires2/setup.cfg b/tests/data/packages/SetupRequires2/setup.cfg new file mode 100644 index 00000000000..6777d6627c6 --- /dev/null +++ b/tests/data/packages/SetupRequires2/setup.cfg @@ -0,0 +1,10 @@ +[metadata] +name = SetupRequires2 +requires-dist = + SetupRequires + +[extras] +a = + simple +b = + simple2 diff --git a/tests/data/packages/SetupRequires2/setup.py b/tests/data/packages/SetupRequires2/setup.py new file mode 100644 index 00000000000..754287ae15c --- /dev/null +++ b/tests/data/packages/SetupRequires2/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='SetupRequires2', + version='0.0.1', + packages=['setuprequires2'], +) diff --git a/tests/data/packages/SetupRequires2/setuprequires2/__init__.py b/tests/data/packages/SetupRequires2/setuprequires2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/packages/SetupRequires3/setup.cfg b/tests/data/packages/SetupRequires3/setup.cfg new file mode 100644 index 00000000000..04f2cec4311 --- /dev/null +++ b/tests/data/packages/SetupRequires3/setup.cfg @@ -0,0 +1,4 @@ +[metadata] +name = SetupRequires3 +requires-dist = + SetupRequires2[a,b] diff --git a/tests/data/packages/SetupRequires3/setup.py b/tests/data/packages/SetupRequires3/setup.py new file mode 100644 index 00000000000..1fea8e52850 --- /dev/null +++ b/tests/data/packages/SetupRequires3/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='SetupRequires3', + version='0.0.1', + packages=['setuprequires3'], +) diff --git a/tests/data/packages/SetupRequires3/setuprequires3/__init__.py b/tests/data/packages/SetupRequires3/setuprequires3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 42652358968..b2a46788f1b 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -732,3 +732,35 @@ def test_install_upgrade_editable_depending_on_other_editable(script): script.pip('install', '--upgrade', '--editable', pkgb_path) result = script.pip('list') assert "pkgb" in result.stdout + + +def _test_setup_requires(script, data, options, name): + to_install = data.packages.join(name) + args = ['install'] + options + [to_install, '-f', data.packages] + res = script.pip(*args, expect_error=False) + assert 'Running setup.py install for upper\n' in str(res) + return res + + +def test_install_declarative_setup_requires_editable(script, data): + res = _test_setup_requires(script, data, ['-e'], 'SetupRequires') + assert 'Running setup.py develop for SetupRequires\n' in str(res), str(res) + + +def test_install_declarative_setup_requires(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires') + assert 'Running setup.py install for SetupRequires\n' in str(res), str(res) + + +def test_install_declarative_requires(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires2') + assert script.site_packages / 'setuprequires2' in res.files_created, res + + +def test_install_declarative_extras(script, data): + res = _test_setup_requires(script, data, [], 'SetupRequires3') + assert 'Running setup.py install for SetupRequires\n' in str(res), str(res) + assert 'Running setup.py install for simple\n' in str(res), str(res) + assert 'Running setup.py install for simple2\n' in str(res), str(res) + assert 'Running setup.py install for SetupRequires3\n' in str(res), \ + str(res) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 47ecbabc3ac..bceaebd7e3b 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -15,8 +15,10 @@ from pip.req import (InstallRequirement, RequirementSet, Requirements, parse_requirements) from pip.req.req_install import parse_editable, _filter_install +from pip.req import req_set from pip.utils import read_text_file from pip._vendor import pkg_resources +from pip._vendor.six.moves import configparser from tests.lib import assert_raises_regexp @@ -460,3 +462,43 @@ def test_filter_install(): INFO, 'foo bar') assert _filter_install('I made a SyntaxError') == ( INFO, 'I made a SyntaxError') + + +def _base_cfg(): + cfg = configparser.SafeConfigParser() + cfg.add_section('metadata') + cfg.set('metadata', 'name', 'foo') + return cfg + + +def test_setup_cfg_no_deps(): + cfg = _base_cfg() + dist = req_set.SetupCfgDistribution(cfg, None) + with pytest.raises(req_set.BadStaticSetupCfg): + req_set._validate_static(dist) + + +def _setup_cfg_has_option(option): + cfg = _base_cfg() + cfg.set('metadata', option, 'dep') + dist = req_set.SetupCfgDistribution(cfg, None) + req_set._validate_static(dist) + + +def _setup_cfg_conflicting_option(option1, option2): + cfg = _base_cfg() + cfg.set('metadata', option1, 'dep') + cfg.set('metadata', option2, 'dep') + dist = req_set.SetupCfgDistribution(cfg, None) + with pytest.raises(req_set.BadStaticSetupCfg): + req_set._validate_static(dist) + + +def test_setup_cfg_setup_requires(): + _setup_cfg_has_option('setup-requires') + + +def test_setup_cfg_install_requires(): + _setup_cfg_has_option('install-requires') + _setup_cfg_has_option('requires-dist') + _setup_cfg_conflicting_option('install-requires', 'requires-dist')