Support declarative requirements.

This adds support per the discussion we had on distutils-sig for
declarative requirements in setup.cfg.

Supported are setup-requires (the problem to be solved).
install-requires, needed as a consequences of supporting setup
requirements, because we can't run egg_info during the
pre-installation phase to detect requires. Similarly extras are
supported, for the same reason.

For compatibility with d2to1 we support requires-dist as an alias
for install-requires in the setup.cfg metadata section.

docs/reference/pip_install.rst

@@ -337,9 +337,58 @@ the project path.  This is one advantage over just using ``setup.py develop``,
 which creates the "egg-info" directly relative the current working directory.
 
 
+Build System Interface
+++++++++++++++++++++++
+
+In order for pip to install a package from source, pip must recognise the build
+system. Today only one build system is recognised, with two variants.
+
+If there is a ``setup.cfg`` with any of ``[extras]``, ``install-requires``,
+``setup-requires``, or ``requires-dist`` present then a declarative setuptools
+package is detected.
+
+Otherwise there is a ``setup.py`` then non-declarative setuptools is assumed.
+
+Declarative setuptools
+~~~~~~~~~~~~~~~~~~~~~~
+
+``setup.py`` must implement the following commands::
+
+    setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX]
+
+With declarative dependencies, easy_install - the ``setup_requires`` keyword
+is never triggered, as pip can take care of installing the requirements before
+``setup.py`` is invoked.
+
+``setup.cfg`` must contain the package name, and one or more of setup requires,
+install requires or extra requires::
+
+    [metadata]
+    name = Example
+    setup-requires =
+      somedep
+    install-requires =
+      runtimedep
+    [extras]
+    one =
+      anotherdep
+    tests =
+      mytestdep
+
+For compatibility with ``d2to1`` ``requires-dist`` is accepted as an alias for
+``install-requires``, though if both are supplied an error will occur.
+
+Non-declarative setuptools
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``setup.py`` must implement the following commands::
+
+    setup.py egg_info [--egg-base XXX]
+    setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX]
 
-Controlling setup_requires
-++++++++++++++++++++++++++
+The ``egg_info`` command should create egg metadata for the package, as
+described in the setuptools documentation at
+http://pythonhosted.org/setuptools/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags
 
 Setuptools offers the ``setup_requires`` `setup() keyword
 <http://pythonhosted.org/setuptools/setuptools.html#new-and-changed-setup-keywords>`_
@@ -370,19 +419,8 @@ To have the dependency located from a local directory and not crawl PyPI, add th
   allow_hosts = ''
   find_links = file:///path/to/local/archives
 
-
-Build System Interface
-++++++++++++++++++++++
-
-In order for pip to install a package from source, ``setup.py`` must implement
-the following commands::
-
-    setup.py egg_info [--egg-base XXX]
-    setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX]
-
-The ``egg_info`` command should create egg metadata for the package, as
-described in the setuptools documentation at
-http://pythonhosted.org/setuptools/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags
+Common setuptools behaviour
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The ``install`` command should implement the complete process of installing the
 package to the target directory XXX.
@@ -404,13 +442,14 @@ Investigate in more detail when this command is required).
 
 No other build system commands are invoked by the ``pip install`` command.
 
+Wheels
+~~~~~~
+
 Installing a package from a wheel does not invoke the build system at all.
 
 .. _PyPI: http://pypi.python.org/pypi/
 .. _setuptools extras: http://packages.python.org/setuptools/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies
 
-
-
 .. _`pip install Options`:
 
 Options

pip/req/req_set.py

@@ -7,6 +7,7 @@
 
 from pip._vendor import pkg_resources
 from pip._vendor import requests
+from pip._vendor.six.moves import configparser
 
 from pip.download import (url_to_path, unpack_url)
 from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled,
@@ -80,6 +81,16 @@ def prep_for_dist(self):
         raise NotImplementedError(self.dist)
 
 
+def _sdist_or_static(req_to_install):
+    result = IsStaticMetadata(req_to_install)
+    try:
+        result.dist(None)
+        return result
+    except (configparser.NoSectionError, configparser.NoOptionError,
+            BadStaticSetupCfg):
+        return IsSDist(req_to_install)
+
+
 def make_abstract_dist(req_to_install):
     """Factory to make an abstract dist object.
 
@@ -89,11 +100,11 @@ def make_abstract_dist(req_to_install):
     :return: A concrete DistAbstraction.
     """
     if req_to_install.editable:
-        return IsSDist(req_to_install)
+        return _sdist_or_static(req_to_install)
     elif req_to_install.link and req_to_install.link.is_wheel:
         return IsWheel(req_to_install)
     else:
-        return IsSDist(req_to_install)
+        return _sdist_or_static(req_to_install)
 
 
 class IsWheel(DistAbstraction):
@@ -123,6 +134,98 @@ def prep_for_dist(self):
         self.req_to_install.assert_source_matches_version()
 
 
+class SetupCfgDistribution(pkg_resources.Distribution):
+    """A Distribution object that consults setup.cfg."""
+
+    def __init__(self, cfg, location=None):
+        self._cfg = cfg
+        project_name = cfg.get('metadata', 'name')
+        super(SetupCfgDistribution, self).__init__(
+            location=location, project_name=project_name)
+
+    @property
+    def _dep_map(self):
+        # Distribution._dep_map is not generic - its expressed in terms of the
+        # requires.txt format from within an egg info directory, and the
+        # metadata interfae within Distribution looks for files-and-lines.
+        # To provide metadata from a ConfigParser we could either marshall
+        # the data to a VFS, or we can reimplement this property which is
+        # the primary worker to obtain requirements.
+        try:
+            return self.__dep_map
+        except AttributeError:
+            # requires
+            requires = self._option('install-requires', 'requires-dist')
+            extras = (
+                self._cfg.has_section('extras')
+                and self._cfg.items('extras')) or []
+            dm = {}
+            dm.setdefault(None, []).extend(
+                pkg_resources.parse_requirements(requires))
+            for extra, extra_reqs in extras:
+                dm.setdefault(extra, []).extend(
+                    pkg_resources.parse_requirements(extra_reqs))
+            self.__dep_map = dm
+            return dm
+
+    def setup_requires(self):
+        try:
+            setup_requires = self._cfg.get('metadata', 'setup-requires')
+        except configparser.NoOptionError:
+            return []
+        return list(pkg_resources.parse_requirements(setup_requires))
+
+    def _option(self, option1, option2):
+        try:
+            result = self._cfg.get('metadata', option1)
+            if self._cfg.has_option('metadata', option2):
+                raise BadStaticSetupCfg('both %s and %s' % (option1, option2))
+            return result
+        except configparser.NoOptionError:
+            try:
+                return self._cfg.get('metadata', option2)
+            except configparser.NoOptionError:
+                return []
+
+
+class BadStaticSetupCfg(Exception):
+    pass
+
+
+def _validate_static(dist):
+    has_deps = False
+    for deps in dist._dep_map.values():
+        if deps:
+            has_deps = True
+            break
+    if not (dist.project_name and (has_deps or dist.setup_requires())):
+        raise BadStaticSetupCfg()
+
+
+class IsStaticMetadata(DistAbstraction):
+    """A static setup.cfg based source tree.
+
+    Must have a name and requires|setup_requires in setup.cfg to be usable.
+    We look for either requires or setup-requires to handle both the case where
+    a setup.py has setup-requires but no runtime requirements are declared, and
+    the case where folk have declaratively expressed their dist requirements
+    without needing a setup-requires.
+    """
+    def dist(self, finder):
+        cfg = configparser.SafeConfigParser()
+        setup_cfg = os.path.join(self.req_to_install.source_dir, 'setup.cfg')
+        cfg.read(setup_cfg)
+        dist = SetupCfgDistribution(cfg=cfg, location=setup_cfg)
+        _validate_static(dist)
+        return dist
+
+    def prep_for_dist(self):
+        self._dist = self.dist(None)
+        self.req_to_install.req = pkg_resources.Requirement.parse(
+            self._dist.project_name)
+        self.req_to_install._correct_build_location()
+
+
 class Installed(DistAbstraction):
 
     def dist(self, finder):
@@ -526,6 +629,15 @@ def add_req(subreq):
                 self.add_requirement(req_to_install)
 
             if not self.ignore_dependencies:
+                if getattr(dist, 'setup_requires', None):
+                    setup_requires = dist.setup_requires()
+                    if setup_requires:
+                        logger.debug(
+                            "Installing setup_requires: %r",
+                            ','.join(r.project_name for r in setup_requires))
+                    for subreq in setup_requires:
+                        add_req(subreq)
+
                 if (req_to_install.extras):
                     logger.debug(
                         "Installing extra requirements: %r",

tests/data/packages/README.txt

@@ -71,6 +71,39 @@ priority-*
 ----------
 used for testing wheel priority over sdists
 
+SetupRequires
+-------------
+
+has a setup.cfg declaring a setup-requires on upper, and a setup.py that will
+fail to import if upper is not installed.
+
+SetupRequires-0.0.1.tar.gz
+--------------------------
+
+SetupRequires sdisted, for testing transitive setup_requires.
+
+SetupRequires2
+--------------
+
+has a setup.cfg declaring a dist-requires on setuprequires. Covers both
+setup-requires in depended-on packages, and setup.cfg with only requires-dist
+expressed.
+Also in setup.cfg declares two extras - a and b, a which brings in simple
+and b which brings in simple2, for testing extras from setup.cfg.
+
+SetupRequires2-0.0.1.tar.gz
+---------------------------
+
+SetupRequires2 sdisted, for testing declarative extras.
+
+SetupRequires3
+--------------
+
+requires SetupRequires2[a,b], as using extras for local paths is currently
+broken (issue 1236). Ideally SetupRequires3 would have the extras itself
+and no requires-dist (to test declarative extras as sole requirements).
+
+
 simple[2]-[123].0.tar.gz
 ------------------------
 contains "simple[2]" package; good for basic testing and version logic.

tests/data/packages/SetupRequires/setup.cfg

@@ -0,0 +1,4 @@
+[metadata]
+name = SetupRequires
+requires-setup =
+  upper

tests/data/packages/SetupRequires/setup.py

@@ -0,0 +1,8 @@
+from setuptools import setup
+import upper
+
+setup(
+    name='SetupRequires',
+    version='0.0.1',
+    packages=['setuprequires'],
+)

tests/data/packages/SetupRequires2/setup.cfg

@@ -0,0 +1,10 @@
+[metadata]
+name = SetupRequires2
+requires-dist =
+  SetupRequires
+
+[extras]
+a = 
+  simple
+b =
+  simple2

tests/data/packages/SetupRequires2/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+    name='SetupRequires2',
+    version='0.0.1',
+    packages=['setuprequires2'],
+)

tests/data/packages/SetupRequires3/setup.cfg

@@ -0,0 +1,4 @@
+[metadata]
+name = SetupRequires3
+requires-dist =
+  SetupRequires2[a,b]

tests/data/packages/SetupRequires3/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+    name='SetupRequires3',
+    version='0.0.1',
+    packages=['setuprequires3'],
+)

tests/functional/test_install.py

@@ -732,3 +732,35 @@ def test_install_upgrade_editable_depending_on_other_editable(script):
     script.pip('install', '--upgrade', '--editable', pkgb_path)
     result = script.pip('list')
     assert "pkgb" in result.stdout
+
+
+def _test_setup_requires(script, data, options, name):
+    to_install = data.packages.join(name)
+    args = ['install'] + options + [to_install, '-f', data.packages]
+    res = script.pip(*args, expect_error=False)
+    assert 'Running setup.py install for upper\n' in str(res)
+    return res
+
+
+def test_install_declarative_setup_requires_editable(script, data):
+    res = _test_setup_requires(script, data, ['-e'], 'SetupRequires')
+    assert 'Running setup.py develop for SetupRequires\n' in str(res), str(res)
+
+
+def test_install_declarative_setup_requires(script, data):
+    res = _test_setup_requires(script, data, [], 'SetupRequires')
+    assert 'Running setup.py install for SetupRequires\n' in str(res), str(res)
+
+
+def test_install_declarative_requires(script, data):
+    res = _test_setup_requires(script, data, [], 'SetupRequires2')
+    assert script.site_packages / 'setuprequires2' in res.files_created, res
+
+
+def test_install_declarative_extras(script, data):
+    res = _test_setup_requires(script, data, [], 'SetupRequires3')
+    assert 'Running setup.py install for SetupRequires\n' in str(res), str(res)
+    assert 'Running setup.py install for simple\n' in str(res), str(res)
+    assert 'Running setup.py install for simple2\n' in str(res), str(res)
+    assert 'Running setup.py install for SetupRequires3\n' in str(res), \
+        str(res)