WIP move to pycapsule

.github/workflows/build.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - main
-      - tkp/linux 
+      - tkp/linux
     tags:
       - v*
     paths-ignore:
@@ -52,11 +52,11 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # for pull_request so we can do HEAD^2
           fetch-depth: 2
-  
+
       - name: Initialize variables
         uses: actions-ext/general/initialize-variables@v1
         id: initialize
@@ -91,7 +91,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -109,7 +109,7 @@ jobs:
 
       # - name: Install dependencies
       #   run: make develop-cpp
-    
+
       - name: Install dependencies
         run: make develop-arrow-ubuntu
         if: ${{ runner.os == 'Linux' }}
@@ -167,7 +167,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run cibuildwheel
         uses: actions-ext/python/run-cibuildwheel@v1
@@ -215,7 +215,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions-ext/python/setup@v1
@@ -228,14 +228,14 @@ jobs:
 
       # - name: Install dependencies
       #   run: make develop-cpp
-    
+
       - name: Install dependencies
         run: make develop-arrow-ubuntu
         if: ${{ runner.os == 'Linux' }}
 
       - name: Install dependencies
         run: make develop-py
-  
+
       - name: Run cibuildwheel
         uses: actions-ext/python/run-cibuildwheel@v1
         with:
@@ -270,10 +270,10 @@ jobs:
           - '3.9'
     runs-on: ${{ matrix.os }}
     if: ${{ needs.initialize.outputs.FULL_BUILD == 'true' }}
- 
+
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions-ext/python/setup@v1
@@ -286,14 +286,14 @@ jobs:
 
       # - name: Install dependencies
       #   run: make develop-cpp
-    
+
       - name: Install dependencies
         run: make develop-arrow-ubuntu
         if: ${{ runner.os == 'Linux' }}
 
       - name: Install dependencies
         run: make develop-py
-  
+
       - name: Build sdist
         run: make dist-sdist
 
@@ -328,7 +328,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions-ext/python/setup@v1
@@ -380,7 +380,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions-ext/python/setup@v1
@@ -393,14 +393,14 @@ jobs:
 
       # - name: Install dependencies
       #   run: make develop-cpp
-    
+
       - name: Install dependencies
         run: make develop-arrow-ubuntu
         if: ${{ runner.os == 'Linux' }}
 
       - name: Install dependencies
         run: make develop-py
-  
+
       - name: Download wheels
         uses: actions-ext/python/download-dist@v1
         with:
@@ -409,7 +409,7 @@ jobs:
 
       - name: Test project
         run: make tests-ci
-  
+
       - name: Upload test results
         uses: EnricoMi/publish-unit-test-result-action@v2
         with:
@@ -457,7 +457,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions-ext/python/setup@v1
@@ -470,22 +470,22 @@ jobs:
 
       # - name: Install dependencies
       #   run: make develop-cpp
-    
+
       - name: Install dependencies
         run: make develop-arrow-ubuntu
         if: ${{ runner.os == 'Linux' }}
 
       - name: Install dependencies
         run: make develop-py
-  
+
       - name: Download wheels
         uses: actions-ext/python/download-dist@v1
         with:
           kind: "sdist"
 
       - name: Test project
         run: make tests-ci
-  
+
       - name: Upload test results
         uses: EnricoMi/publish-unit-test-result-action@v2
         with:

CMakeLists.txt

@@ -121,7 +121,7 @@ if(NOT WIN32)
     if(MACOS)
         set(module_origin_path "@loader_path")
     else()
-        set(module_origin_path "\$ORIGIN")
+      set(module_origin_path "\$ORIGIN")
     endif()
 else()
     set(CMAKE_SHARED_LIBRARY_PREFIX "lib")
@@ -132,7 +132,7 @@ endif()
 # Flags #
 #########
 set(CMAKE_POSITION_INDEPENDENT_CODE On)
-add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
+# add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
 
 # Compiler version flags
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -175,7 +175,7 @@ message("${Red}${CMAKE_MODULE_PATH}${ColorReset}")
 
 
 # broken on mac
-find_package(Arrow REQUIRED)
+find_package(Arrow CONFIG REQUIRED)
 include_directories(${Arrow_INCLUDE_DIR})
 
 ###############################################################################################################
@@ -203,7 +203,7 @@ endif()
 ###############################################################################################################
 # Messages #
 ############
-message("Building CPP Template version v${ARROW_PYTHON_NOCOPY_VERSION_MAJOR}.${ARROW_PYTHON_NOCOPY_VERSION_MINOR}.${ARROW_PYTHON_NOCOPY_VERSION_PATCH} [${ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA}]")
+message("Building Arrow-Python-Nocopy version v${ARROW_PYTHON_NOCOPY_VERSION_MAJOR}.${ARROW_PYTHON_NOCOPY_VERSION_MINOR}.${ARROW_PYTHON_NOCOPY_VERSION_PATCH} [${ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA}]")
 string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LOWER)
 
 if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
@@ -224,7 +224,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 #########################
 set(
     PROJECT_HDRS
-    "${PROJECT_SOURCE_DIR}/src/apn/apn.hpp"
+    "${PROJECT_SOURCE_DIR}/src/apn/apn.h"
     "${PROJECT_SOURCE_DIR}/src/apn/bridge.h"
     "${PROJECT_SOURCE_DIR}/src/apn-common/exports.h"
 )
@@ -235,7 +235,7 @@ set(
 )
 
 add_library(arrow-python-nocopy SHARED ${PROJECT_SRCS})
-target_link_libraries(arrow-python-nocopy PRIVATE ${Arrow_LIBRARY})
+target_link_libraries(arrow-python-nocopy PRIVATE Arrow::arrow_static)
 set_target_properties(arrow-python-nocopy PROPERTIES PUBLIC_HEADER "${PROJECT_HDRS}")
 
 # export symbols
@@ -257,50 +257,21 @@ if(BUILD_PYTHON)
   # Find PyArrow. We will link against it for the build only
   # find_package(Pyarrow REQUIRED)
   # include_directories(${PYARROW_INCLUDE_DIR})
-   
+
   # Find PyBind11
-   find_package(pybind11 REQUIRED)
+   find_package(pybind11 CONFIG REQUIRED)
    include_directories(${pybind11_INCLUDE_DIR})
-   set(VENDORED_PYARROW_ROOT "${PROJECT_SOURCE_DIR}/src/vendored/apache-arrow-12.0.1")
-   include_directories(${VENDORED_PYARROW_ROOT})
-   set(VENDORED_PYARROW_SRCS
-     # ${VENDORED_PYARROW_ROOT}/arrow/python/arrow_to_pandas.cc
-     # ${VENDORED_PYARROW_ROOT}/arrow/python/benchmark.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/common.cc
-     # ${VENDORED_PYARROW_ROOT}/arrow/python/csv.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/datetime.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/decimal.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/deserialize.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/extension_type.cc
-     # ${VENDORED_PYARROW_ROOT}/arrow/python/filesystem.cc
-     # ${VENDORED_PYARROW_ROOT}/arrow/python/flight.cc
-     # ${VENDORED_PYARROW_ROOT}/arrow/python/gdb.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/helpers.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/inference.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/init.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/io.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/ipc.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/numpy_convert.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/numpy_to_arrow.cc
-     # ${VENDORED_PYARROW_ROOT}/arrow/python/parquet_encryption.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/pyarrow.cc
-     # ${VENDORED_PYARROW_ROOT}/arrow/python/python_test.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/python_to_arrow.cc
-     ${VENDORED_PYARROW_ROOT}/arrow/python/serialize.cc
-     # ${VENDORED_PYARROW_ROOT}/arrow/python/udf.cc
-  )
-
+
   # common functionality
   add_library(common SHARED "${PROJECT_SOURCE_DIR}/src/apn-python/common.cpp" ${VENDORED_PYARROW_SRCS})
-  set_target_properties(common PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/common.hpp")
 
   # pybind11 extension
   pybind11_add_module(pybind11extension MODULE "${PROJECT_SOURCE_DIR}/src/apn-python/pybind11.cpp")
-  set_target_properties(pybind11extension PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/pybind11.hpp")
+  set_target_properties(pybind11extension PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/pybind11.h")
 
   # cpython extension
   add_library(cpythonextension SHARED "${PROJECT_SOURCE_DIR}/src/apn-python/cpython.cpp")
-  set_target_properties(cpythonextension PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/cpython.hpp")
+  set_target_properties(cpythonextension PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/cpython.h")
 
   # Link to standalone/common library
   target_link_libraries(common PRIVATE arrow-python-nocopy)

Makefile

@@ -52,7 +52,7 @@ lint-py:
 	python -m ruff arrow_python_nocopy
 
 lint-cpp:
-	clang-format --dry-run -Werror -i -style=file `find ./src -name "*.*pp"` || echo "sometimes flaky"
+	clang-format --dry-run -Werror -i -style=file `find ./src -name "*.{h,cpp}"` || echo "sometimes flaky"
 
 lint: lint-cpp lint-py  ## Run project linters
 
@@ -61,7 +61,7 @@ fix-py:
 	python -m ruff arrow_python_nocopy --fix
 
 fix-cpp:
-	clang-format -i -style=file `find ./src -name "*.*pp"`
+	clang-format -i -style=file `find ./src -name "*.{h,cpp}"`
 
 fix: fix-cpp fix-py  ## Run project autofixers
 
@@ -86,7 +86,7 @@ dist-wheel:  ## Create python wheel dist
 	python setup.py bdist_wheel $(OTHER_ARGS)
 
 dist-cibw:  ## Create python wheel dist with cibuildwheel
-	python -m cibuildwheel --output-dir dist 
+	python -m cibuildwheel --output-dir dist
 
 dist: build dist-sdist dist-wheel  ## Create python dists
 	python -m twine check target/wheels/*

README.md

@@ -2,4 +2,4 @@
 This is a small example of building a C++ library which reads and returns arrays and schemas linked against `libarrow` (but **DOES NOT REQUIRE** `pyarrow`), and building a python extension that uses this library to pass arrays and schemas back and forth.
 
 ## tl;dr
-This utilizes Apache Arrow's **ABI-stable C interface**, which would, for example, allow your combined C++/python extension to compile and link against one version of arrow and be utilized with `pyarrow` which would otherwise be ABI-incompatible.
+This utilizes Apache Arrow's **ABI-stable C interface** and **PyCapsule interface**, which would, for example, allow your combined C++/python extension to compile and link against one version of arrow and be utilized with `pyarrow` which would otherwise be ABI-incompatible.

arrow_python_nocopy/__init__.py

@@ -4,29 +4,13 @@
 import pandas as pd
 from .lib.pybind11extension import array_info, create_array
 from .lib.pybind11extension import schema_info, create_schema
-
-# from .lib.pybind11extension import table_info, create_table
 from .lib.cpythonextension import array_info as array_info_cp, create_array as create_array_cp
 from .lib.cpythonextension import schema_info as schema_info_cp, create_schema as create_schema_cp
 
-# from .lib.cpythonextension import table_info as table_info_cp, create_table as create_table_cp
-
 
 __version__ = "0.1.0"
 
 
-def include_path():
-    return os.path.abspath(os.path.join(os.path.dirname(__file__), "include"))
-
-
-def bin_path():
-    return os.path.abspath(os.path.join(os.path.dirname(__file__), "bin"))
-
-
-def lib_path():
-    return os.path.abspath(os.path.join(os.path.dirname(__file__), "lib"))
-
-
 def _df():
     return pd.DataFrame({"a": pd.Series([1, 2, 3], dtype='Int32'), "b": pd.Series([1.1, 2.2, 3.3], dtype='Float32'), "c": pd.Series(["abc", "def", "ghi"], dtype=str)})
 
@@ -37,23 +21,22 @@
 
 def create_arrow_array_in_python():
     table = _table()
-    array = table.columns['a']
-    print(array_info(array))
+    array = table['a'].combine_chunks()
+    return array_info(array)
 
 
 def create_arrow_array_in_cpp():
-    return create_array()
+    return pa.Array._import_from_c_capsule(*create_array())
 
 
 def create_arrow_schema_in_python():
     table = _table()
     schema = table.schema
-    print(schema_info(schema))
+    return schema_info(schema)
 
 
 def create_arrow_schema_in_cpp():
-    return create_schema()
-
+    return pa.Schema._import_from_c_capsule(create_schema())
 
 # def create_arrow_table_in_python():
 #     table = _table()