Python source reorg (#6867)

* Move python binding sources to src/halide/halide_ * Rename native module to halide_ * Fix tests * Avoid copying Python sources * Fix installation rules * Make diff smaller * trigger buildbots * Add issue todo Co-authored-by: Steven Johnson <[email protected]>
halide · Jul 20, 2022 · 51c06b7 · 51c06b7
1 parent 359026a
commit 51c06b7
Show file tree

Hide file tree

Showing 68 changed files with 76 additions and 12 deletions.
diff --git a/packaging/CMakeLists.txt b/packaging/CMakeLists.txt
@@ -52,8 +52,18 @@ endforeach ()
 if (WITH_PYTHON_BINDINGS)
     set(Halide_INSTALL_PYTHONDIR "${CMAKE_INSTALL_LIBDIR}/python3/site-packages"
         CACHE STRING "Path to Halide Python bindings folder")
+
+    install(DIRECTORY "${Halide_SOURCE_DIR}/python_bindings/src/halide"
+            DESTINATION "${Halide_INSTALL_PYTHONDIR}"
+            COMPONENT Halide_Python
+            FILES_MATCHING
+            PATTERN "*.py"
+            PATTERN "*/halide_" EXCLUDE
+            PATTERN "*/__pycache__" EXCLUDE)
+
     install(TARGETS Halide_Python
-            LIBRARY DESTINATION ${Halide_INSTALL_PYTHONDIR} COMPONENT Halide_Python
+            LIBRARY DESTINATION "${Halide_INSTALL_PYTHONDIR}/halide"
+            COMPONENT Halide_Python
             NAMELINK_COMPONENT Halide_Python)
 endif ()
 
@@ -101,6 +111,13 @@ if (TARGET Halide_Adams2019)
             COMPONENT Halide_Development)
 endif ()
 
+if (TARGET Halide_Python AND NOT CMAKE_INSTALL_RPATH)
+    file(RELATIVE_PATH lib_dir
+        "${CMAKE_CURRENT_BINARY_DIR}/${Halide_INSTALL_PYTHONDIR}/halide"
+        "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+    set_target_properties(Halide_Python PROPERTIES INSTALL_RPATH "${rbase}/${lib_dir}")
+endif ()
+
 ##
 # READMEs and other top-level documentation
 ##
@@ -268,6 +285,11 @@ cpack_add_component(Halide_Development
                     DESCRIPTION "Static Halide libraries and CMake development files"
                     DEPENDS Halide_Runtime)
 
+cpack_add_component(Halide_Python
+                    DISPLAY_NAME "Python bindings"
+                    DESCRIPTION "Python package providing bindings to Halide"
+                    DEPENDS Halide_Runtime)
+
 cpack_add_component(Halide_Documentation
                     DISPLAY_NAME "Halide documentation"
                     DESCRIPTION "Documentation for Halide")
diff --git a/python_bindings/CMakeLists.txt b/python_bindings/CMakeLists.txt
@@ -18,7 +18,7 @@ endif ()
 # Add our sources to this sub-tree.
 ##
 
-add_subdirectory(src)
+add_subdirectory(src/halide)
 add_subdirectory(stub)
 
 option(WITH_TEST_PYTHON "Build Python tests" ON)

diff --git a/python_bindings/src/CMakeLists.txt → python_bindings/src/halide/CMakeLists.txt b/python_bindings/src/CMakeLists.txt → python_bindings/src/halide/CMakeLists.txt
@@ -29,12 +29,13 @@ set(SOURCES
     PyVar.cpp
     PyVarOrRVar.cpp
     )
+list(TRANSFORM SOURCES PREPEND "halide_/")
 
 pybind11_add_module(Halide_Python MODULE ${SOURCES})
 add_library(Halide::Python ALIAS Halide_Python)
 set_target_properties(Halide_Python
                       PROPERTIES
-                      LIBRARY_OUTPUT_NAME halide
+                      LIBRARY_OUTPUT_NAME halide_
                       EXPORT_NAME Python)
 target_link_libraries(Halide_Python PRIVATE Halide::Halide)
 

diff --git a/python_bindings/src/halide/__init__.py b/python_bindings/src/halide/__init__.py
@@ -0,0 +1,24 @@
+# TODO(#6870): The following three lines are a stop-gap. This file should just
+# contain the last two imports, at least until the pure-Python part of the
+# library grows.
+#
+# There are three main reasons why this exists:
+#
+# 1. relative imports cannot be overloaded with sys.path
+# 2. for a variety of reasons, copying the python sources next to the
+#    halide_.*.so module is difficult to make work in 100% of cases in CMake
+# 3. even if we could copy them reliably, copying these files into the build
+#    folder seems inelegant
+#
+# Fortunately, there are apparently other hooks besides sys.path that we could
+# use to redirect a failing relative import.
+#
+# https://docs.python.org/3/reference/import.html#finders-and-loaders
+# https://github.com/halide/Halide/issues/6870
+
+import sys
+from pathlib import Path
+sys.path.append(str(Path(__file__).parent.resolve(strict=True)))
+
+from halide_ import *
+from halide_ import _, _1, _2, _3, _4, _5, _6, _7, _8, _9
diff --git a/python_bindings/src/PyArgument.cpp → ...indings/src/halide/halide_/PyArgument.cpp b/python_bindings/src/PyArgument.cpp → ...indings/src/halide/halide_/PyArgument.cpp
diff --git a/python_bindings/src/PyArgument.h → ..._bindings/src/halide/halide_/PyArgument.h b/python_bindings/src/PyArgument.h → ..._bindings/src/halide/halide_/PyArgument.h
diff --git a/python_bindings/src/PyBinaryOperators.h → ...gs/src/halide/halide_/PyBinaryOperators.h b/python_bindings/src/PyBinaryOperators.h → ...gs/src/halide/halide_/PyBinaryOperators.h
diff --git a/python_bindings/src/PyBoundaryConditions.cpp → ...c/halide/halide_/PyBoundaryConditions.cpp b/python_bindings/src/PyBoundaryConditions.cpp → ...c/halide/halide_/PyBoundaryConditions.cpp
diff --git a/python_bindings/src/PyBoundaryConditions.h → ...src/halide/halide_/PyBoundaryConditions.h b/python_bindings/src/PyBoundaryConditions.h → ...src/halide/halide_/PyBoundaryConditions.h
diff --git a/python_bindings/src/PyBuffer.cpp → ..._bindings/src/halide/halide_/PyBuffer.cpp b/python_bindings/src/PyBuffer.cpp → ..._bindings/src/halide/halide_/PyBuffer.cpp
diff --git a/python_bindings/src/PyBuffer.h → ...on_bindings/src/halide/halide_/PyBuffer.h b/python_bindings/src/PyBuffer.h → ...on_bindings/src/halide/halide_/PyBuffer.h
diff --git a/python_bindings/src/PyCallable.cpp → ...indings/src/halide/halide_/PyCallable.cpp b/python_bindings/src/PyCallable.cpp → ...indings/src/halide/halide_/PyCallable.cpp
diff --git a/python_bindings/src/PyCallable.h → ..._bindings/src/halide/halide_/PyCallable.h b/python_bindings/src/PyCallable.h → ..._bindings/src/halide/halide_/PyCallable.h
diff --git a/python_bindings/src/PyConciseCasts.cpp → ...ngs/src/halide/halide_/PyConciseCasts.cpp b/python_bindings/src/PyConciseCasts.cpp → ...ngs/src/halide/halide_/PyConciseCasts.cpp
diff --git a/python_bindings/src/PyConciseCasts.h → ...dings/src/halide/halide_/PyConciseCasts.h b/python_bindings/src/PyConciseCasts.h → ...dings/src/halide/halide_/PyConciseCasts.h
diff --git a/python_bindings/src/PyDerivative.cpp → ...dings/src/halide/halide_/PyDerivative.cpp b/python_bindings/src/PyDerivative.cpp → ...dings/src/halide/halide_/PyDerivative.cpp
diff --git a/python_bindings/src/PyDerivative.h → ...indings/src/halide/halide_/PyDerivative.h b/python_bindings/src/PyDerivative.h → ...indings/src/halide/halide_/PyDerivative.h
diff --git a/python_bindings/src/PyEnums.cpp → ...n_bindings/src/halide/halide_/PyEnums.cpp b/python_bindings/src/PyEnums.cpp → ...n_bindings/src/halide/halide_/PyEnums.cpp
diff --git a/python_bindings/src/PyEnums.h → python_bindings/src/halide/halide_/PyEnums.h b/python_bindings/src/PyEnums.h → python_bindings/src/halide/halide_/PyEnums.h
diff --git a/python_bindings/src/PyError.cpp → ...n_bindings/src/halide/halide_/PyError.cpp b/python_bindings/src/PyError.cpp → ...n_bindings/src/halide/halide_/PyError.cpp
diff --git a/python_bindings/src/PyError.h → python_bindings/src/halide/halide_/PyError.h b/python_bindings/src/PyError.h → python_bindings/src/halide/halide_/PyError.h
diff --git a/python_bindings/src/PyExpr.cpp → ...on_bindings/src/halide/halide_/PyExpr.cpp b/python_bindings/src/PyExpr.cpp → ...on_bindings/src/halide/halide_/PyExpr.cpp
diff --git a/python_bindings/src/PyExpr.h → python_bindings/src/halide/halide_/PyExpr.h b/python_bindings/src/PyExpr.h → python_bindings/src/halide/halide_/PyExpr.h
diff --git a/python_bindings/src/PyExternFuncArgument.cpp → ...c/halide/halide_/PyExternFuncArgument.cpp b/python_bindings/src/PyExternFuncArgument.cpp → ...c/halide/halide_/PyExternFuncArgument.cpp
diff --git a/python_bindings/src/PyExternFuncArgument.h → ...src/halide/halide_/PyExternFuncArgument.h b/python_bindings/src/PyExternFuncArgument.h → ...src/halide/halide_/PyExternFuncArgument.h
diff --git a/python_bindings/src/PyFunc.cpp → ...on_bindings/src/halide/halide_/PyFunc.cpp b/python_bindings/src/PyFunc.cpp → ...on_bindings/src/halide/halide_/PyFunc.cpp
diff --git a/python_bindings/src/PyFunc.h → python_bindings/src/halide/halide_/PyFunc.h b/python_bindings/src/PyFunc.h → python_bindings/src/halide/halide_/PyFunc.h
diff --git a/python_bindings/src/PyFuncRef.cpp → ...bindings/src/halide/halide_/PyFuncRef.cpp b/python_bindings/src/PyFuncRef.cpp → ...bindings/src/halide/halide_/PyFuncRef.cpp
diff --git a/python_bindings/src/PyFuncRef.h → ...n_bindings/src/halide/halide_/PyFuncRef.h b/python_bindings/src/PyFuncRef.h → ...n_bindings/src/halide/halide_/PyFuncRef.h
diff --git a/python_bindings/src/PyHalide.cpp → ..._bindings/src/halide/halide_/PyHalide.cpp b/python_bindings/src/PyHalide.cpp → ..._bindings/src/halide/halide_/PyHalide.cpp
@@ -34,7 +34,7 @@ static_assert(PY_VERSION_HEX >= 0x03000000,
               "We appear to be compiling against Python 2.x rather than 3.x, which is not supported.");
 
 #ifndef HALIDE_PYBIND_MODULE_NAME
-#define HALIDE_PYBIND_MODULE_NAME halide
+#define HALIDE_PYBIND_MODULE_NAME halide_
 #endif
 
 PYBIND11_MODULE(HALIDE_PYBIND_MODULE_NAME, m) {

diff --git a/python_bindings/src/PyHalide.h → ...on_bindings/src/halide/halide_/PyHalide.h b/python_bindings/src/PyHalide.h → ...on_bindings/src/halide/halide_/PyHalide.h
diff --git a/python_bindings/src/PyIROperator.cpp → ...dings/src/halide/halide_/PyIROperator.cpp b/python_bindings/src/PyIROperator.cpp → ...dings/src/halide/halide_/PyIROperator.cpp
diff --git a/python_bindings/src/PyIROperator.h → ...indings/src/halide/halide_/PyIROperator.h b/python_bindings/src/PyIROperator.h → ...indings/src/halide/halide_/PyIROperator.h
diff --git a/python_bindings/src/PyImageParam.cpp → ...dings/src/halide/halide_/PyImageParam.cpp b/python_bindings/src/PyImageParam.cpp → ...dings/src/halide/halide_/PyImageParam.cpp
diff --git a/python_bindings/src/PyImageParam.h → ...indings/src/halide/halide_/PyImageParam.h b/python_bindings/src/PyImageParam.h → ...indings/src/halide/halide_/PyImageParam.h
diff --git a/python_bindings/src/PyInlineReductions.cpp → ...src/halide/halide_/PyInlineReductions.cpp b/python_bindings/src/PyInlineReductions.cpp → ...src/halide/halide_/PyInlineReductions.cpp
diff --git a/python_bindings/src/PyInlineReductions.h → ...s/src/halide/halide_/PyInlineReductions.h b/python_bindings/src/PyInlineReductions.h → ...s/src/halide/halide_/PyInlineReductions.h
diff --git a/python_bindings/src/PyLambda.cpp → ..._bindings/src/halide/halide_/PyLambda.cpp b/python_bindings/src/PyLambda.cpp → ..._bindings/src/halide/halide_/PyLambda.cpp
diff --git a/python_bindings/src/PyLambda.h → ...on_bindings/src/halide/halide_/PyLambda.h b/python_bindings/src/PyLambda.h → ...on_bindings/src/halide/halide_/PyLambda.h
diff --git a/python_bindings/src/PyLoopLevel.cpp → ...ndings/src/halide/halide_/PyLoopLevel.cpp b/python_bindings/src/PyLoopLevel.cpp → ...ndings/src/halide/halide_/PyLoopLevel.cpp
diff --git a/python_bindings/src/PyLoopLevel.h → ...bindings/src/halide/halide_/PyLoopLevel.h b/python_bindings/src/PyLoopLevel.h → ...bindings/src/halide/halide_/PyLoopLevel.h
diff --git a/python_bindings/src/PyMachineParams.cpp → ...gs/src/halide/halide_/PyMachineParams.cpp b/python_bindings/src/PyMachineParams.cpp → ...gs/src/halide/halide_/PyMachineParams.cpp
diff --git a/python_bindings/src/PyMachineParams.h → ...ings/src/halide/halide_/PyMachineParams.h b/python_bindings/src/PyMachineParams.h → ...ings/src/halide/halide_/PyMachineParams.h
diff --git a/python_bindings/src/PyModule.cpp → ..._bindings/src/halide/halide_/PyModule.cpp b/python_bindings/src/PyModule.cpp → ..._bindings/src/halide/halide_/PyModule.cpp
diff --git a/python_bindings/src/PyModule.h → ...on_bindings/src/halide/halide_/PyModule.h b/python_bindings/src/PyModule.h → ...on_bindings/src/halide/halide_/PyModule.h
diff --git a/python_bindings/src/PyParam.cpp → ...n_bindings/src/halide/halide_/PyParam.cpp b/python_bindings/src/PyParam.cpp → ...n_bindings/src/halide/halide_/PyParam.cpp
diff --git a/python_bindings/src/PyParam.h → python_bindings/src/halide/halide_/PyParam.h b/python_bindings/src/PyParam.h → python_bindings/src/halide/halide_/PyParam.h
diff --git a/python_bindings/src/PyPipeline.cpp → ...indings/src/halide/halide_/PyPipeline.cpp b/python_bindings/src/PyPipeline.cpp → ...indings/src/halide/halide_/PyPipeline.cpp
diff --git a/python_bindings/src/PyPipeline.h → ..._bindings/src/halide/halide_/PyPipeline.h b/python_bindings/src/PyPipeline.h → ..._bindings/src/halide/halide_/PyPipeline.h
diff --git a/python_bindings/src/PyRDom.cpp → ...on_bindings/src/halide/halide_/PyRDom.cpp b/python_bindings/src/PyRDom.cpp → ...on_bindings/src/halide/halide_/PyRDom.cpp
diff --git a/python_bindings/src/PyRDom.h → python_bindings/src/halide/halide_/PyRDom.h b/python_bindings/src/PyRDom.h → python_bindings/src/halide/halide_/PyRDom.h
diff --git a/python_bindings/src/PyScheduleMethods.h → ...gs/src/halide/halide_/PyScheduleMethods.h b/python_bindings/src/PyScheduleMethods.h → ...gs/src/halide/halide_/PyScheduleMethods.h
diff --git a/python_bindings/src/PyStage.cpp → ...n_bindings/src/halide/halide_/PyStage.cpp b/python_bindings/src/PyStage.cpp → ...n_bindings/src/halide/halide_/PyStage.cpp
diff --git a/python_bindings/src/PyStage.h → python_bindings/src/halide/halide_/PyStage.h b/python_bindings/src/PyStage.h → python_bindings/src/halide/halide_/PyStage.h
diff --git a/python_bindings/src/PyTarget.cpp → ..._bindings/src/halide/halide_/PyTarget.cpp b/python_bindings/src/PyTarget.cpp → ..._bindings/src/halide/halide_/PyTarget.cpp
diff --git a/python_bindings/src/PyTarget.h → ...on_bindings/src/halide/halide_/PyTarget.h b/python_bindings/src/PyTarget.h → ...on_bindings/src/halide/halide_/PyTarget.h
diff --git a/python_bindings/src/PyTuple.cpp → ...n_bindings/src/halide/halide_/PyTuple.cpp b/python_bindings/src/PyTuple.cpp → ...n_bindings/src/halide/halide_/PyTuple.cpp
diff --git a/python_bindings/src/PyTuple.h → python_bindings/src/halide/halide_/PyTuple.h b/python_bindings/src/PyTuple.h → python_bindings/src/halide/halide_/PyTuple.h
diff --git a/python_bindings/src/PyType.cpp → ...on_bindings/src/halide/halide_/PyType.cpp b/python_bindings/src/PyType.cpp → ...on_bindings/src/halide/halide_/PyType.cpp
diff --git a/python_bindings/src/PyType.h → python_bindings/src/halide/halide_/PyType.h b/python_bindings/src/PyType.h → python_bindings/src/halide/halide_/PyType.h
diff --git a/python_bindings/src/PyVar.cpp → python_bindings/src/halide/halide_/PyVar.cpp b/python_bindings/src/PyVar.cpp → python_bindings/src/halide/halide_/PyVar.cpp
diff --git a/python_bindings/src/PyVar.h → python_bindings/src/halide/halide_/PyVar.h b/python_bindings/src/PyVar.h → python_bindings/src/halide/halide_/PyVar.h
diff --git a/python_bindings/src/PyVarOrRVar.cpp → ...ndings/src/halide/halide_/PyVarOrRVar.cpp b/python_bindings/src/PyVarOrRVar.cpp → ...ndings/src/halide/halide_/PyVarOrRVar.cpp
diff --git a/python_bindings/src/PyVarOrRVar.h → ...bindings/src/halide/halide_/PyVarOrRVar.h b/python_bindings/src/PyVarOrRVar.h → ...bindings/src/halide/halide_/PyVarOrRVar.h
diff --git a/python_bindings/test/apps/CMakeLists.txt b/python_bindings/test/apps/CMakeLists.txt
@@ -5,8 +5,14 @@ set(SCRIPTS
     interpolate.py
     local_laplacian.py)
 
+make_shell_path(
+    PYTHONPATH
+    "$<TARGET_FILE_DIR:Halide::Python>"
+    "${Halide_SOURCE_DIR}/python_bindings/src"
+)
+
 set(TEST_ENV
-    "PYTHONPATH=$<SHELL_PATH:$<TARGET_FILE_DIR:Halide::Python>>"
+    "PYTHONPATH=${PYTHONPATH}"
     "HL_TARGET=${Halide_TARGET}"
     "TEST_TMPDIR=$<SHELL_PATH:${CMAKE_CURRENT_BINARY_DIR}>"
     "TEST_IMAGES_DIR=$<SHELL_PATH:${CMAKE_CURRENT_SOURCE_DIR}/../../../apps/images>"

diff --git a/python_bindings/test/correctness/CMakeLists.txt b/python_bindings/test/correctness/CMakeLists.txt
@@ -27,11 +27,13 @@ set(TESTS
     )
 
 # Use generator expressions to get the true output paths of these files.
-make_shell_path(PYTHONPATH
-                "$<TARGET_FILE_DIR:py_aot_bit>"
-                "$<TARGET_FILE_DIR:py_stub_bit>"
-                "$<TARGET_FILE_DIR:Halide::Python>"
-                )
+make_shell_path(
+    PYTHONPATH
+    "$<TARGET_FILE_DIR:py_aot_bit>"
+    "$<TARGET_FILE_DIR:py_stub_bit>"
+    "$<TARGET_FILE_DIR:Halide::Python>"
+    "${Halide_SOURCE_DIR}/python_bindings/src"
+)
 
 foreach (TEST IN LISTS TESTS)
     get_filename_component(TEST_NAME ${TEST} NAME_WE)

diff --git a/python_bindings/tutorial/CMakeLists.txt b/python_bindings/tutorial/CMakeLists.txt
@@ -16,7 +16,12 @@ set(TESTS
     lesson_14_types.py
     )
 
-make_shell_path(PYTHONPATH "$<TARGET_FILE_DIR:lesson_10_halide>" "$<TARGET_FILE_DIR:Halide::Python>")
+make_shell_path(
+    PYTHONPATH
+    "$<TARGET_FILE_DIR:lesson_10_halide>"
+    "$<TARGET_FILE_DIR:Halide::Python>"
+    "${Halide_SOURCE_DIR}/python_bindings/src"
+)
 
 foreach (TEST IN LISTS TESTS)
     get_filename_component(TEST_NAME ${TEST} NAME_WE)

diff --git a/src/autoschedulers/li2018/CMakeLists.txt b/src/autoschedulers/li2018/CMakeLists.txt
@@ -44,7 +44,11 @@ if (WITH_PYTHON_BINDINGS)
         add_test(NAME gradient_autoscheduler_test_py
                  COMMAND Python3::Interpreter "${CMAKE_CURRENT_SOURCE_DIR}/test.py")
 
-        set(PYTHONPATH "$<SHELL_PATH:$<TARGET_FILE_DIR:Halide::Python>>")
+        make_shell_path(
+            PYTHONPATH
+            "$<TARGET_FILE_DIR:Halide::Python>"
+            "${Halide_SOURCE_DIR}/python_bindings/src"
+        )
 
         if (WIN32)
             set(SEP "\\$<SEMICOLON>")