Adds a sage_build_py command in setup.py

This uses the reworked find_extra_files, along with a (currently hard-coded)
data structure more or less identical to the distutils package_data argument,
to copy supplemental non-Python files into the installed sage
package/sub-packages.  This list is also passed to to cleanup_stale_files as
with the extra files found during the Cythonization step.

Most of this could be avoided if we just used package_data directly but
supposedly that's broken in some case.

src/module_list.py

@@ -5,6 +5,12 @@
 
 SAGE_INC = os.path.join(SAGE_LOCAL, 'include')
 
+#########################################################
+### additional per-package data files
+#########################################################
+package_data = {}
+
+
 #########################################################
 ### pkg-config setup
 #########################################################

src/sage_setup/clean.py

@@ -73,10 +73,14 @@ def _find_stale_files(site_packages, python_packages, python_modules, ext_module
 
         sage: from sage.env import SAGE_SRC, SAGE_LIB, SAGE_CYTHONIZED
         sage: from sage_setup.find import find_python_sources, find_extra_files
+        sage: from module_list import package_data
         sage: python_packages, python_modules = find_python_sources(
         ....:     SAGE_SRC, ['sage', 'sage_setup'])
         sage: extra_files = find_extra_files(python_packages, SAGE_SRC,
         ....:     SAGE_CYTHONIZED, ["ntlwrap.cpp"])
+        sage: for package, patterns in package_data.items():
+        ....:     extra_files += find_extra_files(package, SAGE_SRC,
+        ....:         special_filenames=patterns)
         sage: from sage_setup.clean import _find_stale_files
 
     TODO: move ``module_list.py`` into ``sage_setup`` and also check

src/setup.py

@@ -5,6 +5,7 @@
 from distutils import log
 from distutils.core import setup, DistutilsSetupError
 from distutils.command.build_ext import build_ext
+from distutils.command.build_py import build_py
 from distutils.command.install import install
 from distutils.dep_util import newer_group
 
@@ -61,7 +62,7 @@ def excepthook(*exc):
 ### the same directory as this file
 #########################################################
 
-from module_list import ext_modules, library_order, aliases
+from module_list import ext_modules, library_order, aliases, package_data
 from sage.env import *
 
 #########################################################
@@ -655,12 +656,12 @@ def copy_extra_files(self):
         be installed in site-packages/sage.
         """
         dist = self.distribution
-        from sage_setup.find import find_extra_files
         self.cythonized_files = find_extra_files(dist.packages,
             ".", SAGE_CYTHONIZED, ["ntlwrap.cpp"])
 
         for (dst_dir, src_files) in self.cythonized_files:
             dst = os.path.join(self.build_lib, dst_dir)
+            self.mkpath(dst)
             for src in src_files:
                 self.copy_file(src, dst, preserve_mode=False)
 
@@ -681,15 +682,37 @@ def copy_extra_files(self):
 
 print("Discovering Python/Cython source code....")
 t = time.time()
-from sage_setup.find import find_python_sources
+from sage_setup.find import find_python_sources, find_extra_files
 python_packages, python_modules = find_python_sources(
     SAGE_SRC, ['sage', 'sage_setup'])
 
 log.info('python_packages = {0}'.format(python_packages))
-
 print("Discovered Python/Cython sources, time: %.2f seconds." % (time.time() - t))
 
 
+class sage_build_py(build_py):
+    # Find additional data files used by some packages
+    # This information would best be moved out to per-package configuration data
+
+    def finalize_options(self):
+        build_py.finalize_options(self)
+        self.extra_files = []
+
+    def run(self):
+        build_py.run(self)
+        self.copy_extra_files()
+
+    def copy_extra_files(self):
+        for pkg, patterns in package_data.items():
+            for dst_dir, filenames in find_extra_files(
+                    pkg, '.', special_filenames=patterns):
+                self.extra_files.append((dst_dir, filenames))
+                dst = os.path.join(self.build_lib, dst_dir)
+                self.mkpath(dst)
+                for filename in filenames:
+                    self.copy_file(filename, dst, preserve_mode=False)
+
+
 #########################################################
 ### Install Jupyter kernel spec and clean stale files
 #########################################################
@@ -749,7 +772,7 @@ def clean_stale_files(self):
                     dist.packages,
                     py_modules,
                     dist.ext_modules,
-                    cmd_build_ext.cythonized_files)
+                    cmd_build_ext.cythonized_files + cmd_build_py.extra_files)
 
 
 #########################################################
@@ -764,5 +787,6 @@ def clean_stale_files(self):
       author_email= 'http://groups.google.com/group/sage-support',
       url         = 'http://www.sagemath.org',
       packages    = python_packages,
-      cmdclass = dict(build_ext=sage_build_ext, install=sage_install),
+      cmdclass = dict(build_ext=sage_build_ext, build_py=sage_build_py,
+                      install=sage_install),
       ext_modules = ext_modules)