feat(python): Add ArrowDeviceArray extension to Python bindings (#313)

This PR adds basic support for wrapping the `ArrowDeviceArray` in
nanoarrow Python. You have to try pretty hard to get anything that isn't
a CPU array here, but 99% of this is just to get a `repr()` and make the
device array easier to debug/understand. In some future the CUDA and
Metal implementations could live in separate Python packages as well.

```python
import nanoarrow.device
import pyarrow as pa

nanoarrow.device.device_array(
    pa.record_batch([pa.array([1, 2, 3])], ["col"])
)
```

```
<nanoarrow.device.DeviceArray>
- device_type: 1
- device_id: 0
- array: <nanoarrow.Array struct>
  - length: 3
  - offset: 0
  - null_count: 0
  - buffers: (0,)
  - dictionary: NULL
  - children[1]:
    'col': <nanoarrow.Array int64>
      - length: 3
      - offset: 0
      - null_count: 0
      - buffers: (0, 5698482274496)
      - dictionary: NULL
      - children[0]:
```

Most of the code changes here are to add the appropriate reprs for the
`Array` and `Schema` objects. These should work for both CPU and Device
flavours of the C Data interface.

@jorisvandenbossche the reprs should make it easier for your
`__arrow_c_array__` / `__arrow_c_schema__` testing!

---------

Co-authored-by: Joris Van den Bossche <[email protected]>

python/.gitignore

@@ -18,6 +18,7 @@
 
 src/nanoarrow/nanoarrow.c
 src/nanoarrow/nanoarrow.h
+src/nanoarrow/nanoarrow_device.h
 src/nanoarrow/nanoarrow_c.pxd
 src/nanoarrow/*.c
 

python/bootstrap.py

@@ -159,7 +159,19 @@ def copy_or_generate_nanoarrow_c():
 
     maybe_nanoarrow_h = os.path.join(this_dir, "src/nanoarrow/nanoarrow.h")
     maybe_nanoarrow_c = os.path.join(this_dir, "src/nanoarrow/nanoarrow.c")
-    for f in (maybe_nanoarrow_c, maybe_nanoarrow_h):
+    maybe_nanoarrow_device_h = os.path.join(
+        this_dir, "src/nanoarrow/nanoarrow_device.h"
+    )
+    maybe_nanoarrow_device_c = os.path.join(
+        this_dir, "src/nanoarrow/nanoarrow_device.c"
+    )
+
+    for f in (
+        maybe_nanoarrow_c,
+        maybe_nanoarrow_h,
+        maybe_nanoarrow_device_h,
+        maybe_nanoarrow_device_c,
+    ):
         if os.path.exists(f):
             os.unlink(f)
 
@@ -170,6 +182,17 @@ def copy_or_generate_nanoarrow_c():
     has_cmake = os.system("cmake --version") == 0
     build_dir = os.path.join(this_dir, "_cmake")
 
+    if is_in_nanoarrow_repo:
+        device_ext_src = os.path.join(
+            source_dir, "extensions/nanoarrow_device/src/nanoarrow"
+        )
+        shutil.copyfile(
+            os.path.join(device_ext_src, "nanoarrow_device.h"), maybe_nanoarrow_device_h
+        )
+        shutil.copyfile(
+            os.path.join(device_ext_src, "nanoarrow_device.c"), maybe_nanoarrow_device_c
+        )
+
     if has_cmake and is_cmake_dir and is_in_nanoarrow_repo:
         try:
             os.mkdir(build_dir)

python/setup.py

@@ -33,14 +33,18 @@
 
 
 # Set some extra flags for compiling with coverage support
-if os.getenv("NANOARROW_PYTHON_COVERAGE") == "1":
-    coverage_compile_args = ["--coverage"]
-    coverage_link_args = ["--coverage"]
-    coverage_define_macros = [("CYTHON_TRACE", 1)]
+if os.getenv("NANOARROW_COVERAGE") == "1":
+    extra_compile_args = ["--coverage"]
+    extra_link_args = ["--coverage"]
+    extra_define_macros = [("CYTHON_TRACE", 1)]
+elif os.getenv("NANOARROW_DEBUG_EXTENSION") == "1":
+    extra_compile_args = ["-g", "-O0"]
+    extra_link_args = []
+    extra_define_macros = []
 else:
-    coverage_compile_args = []
-    coverage_link_args = []
-    coverage_define_macros = []
+    extra_compile_args = []
+    extra_link_args = []
+    extra_define_macros = []
 
 setup(
     ext_modules=[
@@ -51,10 +55,11 @@
             sources=[
                 "src/nanoarrow/_lib.pyx",
                 "src/nanoarrow/nanoarrow.c",
+                "src/nanoarrow/nanoarrow_device.c",
             ],
-            extra_compile_args=coverage_compile_args,
-            extra_link_args=coverage_link_args,
-            define_macros=coverage_define_macros,
+            extra_compile_args=extra_compile_args,
+            extra_link_args=extra_link_args,
+            define_macros=extra_define_macros,
         )
     ]
 )

python/src/nanoarrow/__init__.py

@@ -16,4 +16,4 @@
 # under the License.
 
 from ._lib import Array, ArrayStream, ArrayView, Schema, c_version  # noqa: F401
-from .lib import array, array_stream, schema  # noqa: F401
+from .lib import array, array_stream, schema, array_view  # noqa: F401