PROTON-????: Support building python extension with unbundled proton

There are 3 cases:
* Bundled proton library - this should be the default and is what you
  should use with pip etc.
* System proton library - this is what you should use in a distro
  package - esp. Fedora/RHEL pr Debian too?
* In dev tree - should build with the in dev tree proton lib for
  debugging.

python/CMakeLists.txt

@@ -53,6 +53,8 @@ set(py_dist_files
     README.rst
     MANIFEST.in
     ext_build.py
+    ext_build_devtree.py
+    ext_build_unbundled.py
     cproton.h
     cproton_ext.c
     cproton.py
@@ -128,7 +130,6 @@ add_custom_command(OUTPUT .timestamp.copied_pysrc
                    COMMAND ${CMAKE_COMMAND} -E copy ${PN_C_SOURCE_DIR}/core/frame_generators.h src/core
                    COMMAND ${CMAKE_COMMAND} -E copy ${PN_C_SOURCE_DIR}/core/frame_consumers.c  src/core
                    COMMAND ${CMAKE_COMMAND} -E copy ${PN_C_SOURCE_DIR}/core/frame_consumers.h  src/core
-                   COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/VERSION.txt            .
                    COMMAND ${CMAKE_COMMAND} -E touch .timestamp.copied_pysrc
                    DEPENDS generated_c_files ${py_cgen} ${py_csrc} ${py_cinc} ${PROJECT_SOURCE_DIR}/VERSION.txt)
 
@@ -139,23 +140,37 @@ foreach(file IN LISTS py_dist_files pysrc)
   list(APPEND pysrc_files "${CMAKE_CURRENT_BINARY_DIR}/${file}")
 endforeach()
 
-add_custom_target(pysrc_copied ALL DEPENDS .timestamp.copied_pysrc ${pysrc_files})
+add_custom_command(OUTPUT VERSION.txt
+                  COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/VERSION.txt .
+                  DEPENDS ${PROJECT_SOURCE_DIR}/VERSION.txt)
+
+add_custom_target(pysrc_copied DEPENDS ${pysrc_files} VERSION.txt)
+add_custom_target(pypkg_src_copied ALL DEPENDS pysrc_copied .timestamp.copied_pysrc)
 
 add_custom_command(OUTPUT ./tox.ini
                    COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/tox.ini" tox.ini
                    DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/tox.ini")
 
+option(BUILD_PYTHON_UNBUNDLED_PKG "Build Python package without bundling qpid-proton-core library" Off)
 # Make python source and binary packages if we have prerequisites
 check_python_module("setuptools" SETUPTOOLS_MODULE_FOUND)
 check_python_module("wheel" WHEEL_MODULE_FOUND)
 check_python_module("build" BUILD_MODULE_FOUND)
 check_python_module("cffi" CFFI_MODULE_FOUND)
 if (BUILD_MODULE_FOUND AND SETUPTOOLS_MODULE_FOUND AND WHEEL_MODULE_FOUND AND CFFI_MODULE_FOUND)
-  add_custom_command(OUTPUT .timestamp.dist
-    DEPENDS pysrc_copied
-    COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
-    COMMAND ${Python_EXECUTABLE} -m build -n
-    COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
+  if (BUILD_PYTHON_UNBUNDLED_PKG)
+    add_custom_command(OUTPUT .timestamp.dist
+      DEPENDS pypkg_src_copied
+      COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
+      COMMAND ${CMAKE_COMMAND} -E env QPID_PYTHON_UNBUNDLED=yes ${Python_EXECUTABLE} -m build -n
+      COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
+  else ()
+    add_custom_command(OUTPUT .timestamp.dist
+      DEPENDS pypkg_src_copied
+      COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
+      COMMAND ${Python_EXECUTABLE} -m build -n
+      COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
+  endif ()
   add_custom_target(pydist ALL DEPENDS .timestamp.dist)
 endif ()
 
@@ -210,9 +225,9 @@ if (BUILD_TESTING)
   add_custom_command(
     OUTPUT .timestamp.cproton_ffi
     COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.cproton_ffi
-    COMMAND ${pytest_executable} ext_build.py
+    COMMAND ${CMAKE_COMMAND} -E env "CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}" ${pytest_executable} ext_build_devtree.py
     COMMAND ${CMAKE_COMMAND} -E touch .timestamp.cproton_ffi
-    DEPENDS ${pytest_venv}/env.txt pysrc_copied
+    DEPENDS ${pytest_venv}/env.txt pysrc_copied qpid-proton-core
   )
 
   pn_add_test(

python/MANIFEST.in

@@ -18,6 +18,7 @@
 
 include VERSION.txt
 include ext_build.py
+include ext_build_unbundled.py
 include cproton.h
 include cproton_ext.c
 include cproton.py

python/ext_build_devtree.py

@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+from cffi import FFI
+
+bld_tree_top = os.environ.get("CMAKE_BINARY_DIR")
+
+cdefs = open('cproton.h').read()
+c_code = open('cproton_ext.c').read()
+
+ffibuilder = FFI()
+ffibuilder.cdef(cdefs)
+ffibuilder.set_source(
+    "cproton_ffi",
+    c_code,
+    include_dirs=[f"{bld_tree_top}/c/include"],
+    libraries=["qpid-proton-core"],
+    extra_link_args=[f"-Wl,-rpath={bld_tree_top}/c"],
+)
+
+if __name__ == "__main__":
+    ffibuilder.compile(verbose=True)

python/ext_build_unbundled.py

@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import cffi.pkgconfig
+
+from cffi import FFI
+
+ffibuilder = FFI()
+
+# cdef() expects a single string declaring the C types, functions and
+# globals needed to use the shared object. It must be in valid C syntax
+# with cffi extensions
+cdefs = open('cproton.h').read()
+ffibuilder.cdef(cdefs)
+
+cffi.pkgconfig.flags_from_pkgconfig(['libqpid-proton-core'])
+
+c_code = open('cproton_ext.c').read()
+ffibuilder.set_source_pkgconfig(
+    "cproton_ffi",
+    ['libqpid-proton-core'],
+    c_code,
+)
+
+if __name__ == "__main__":
+    ffibuilder.compile(verbose=True)

python/setup.py

@@ -16,8 +16,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import os
 from setuptools import setup
 
-setup(
-    cffi_modules='ext_build.py:ffibuilder'
-)
+unbundled = os.environ.get("QPID_PYTHON_UNBUNDLED")
+
+if unbundled:
+    setup(
+        cffi_modules='ext_build_unbundled.py:ffibuilder'
+    )
+else:
+    setup(
+        cffi_modules='ext_build.py:ffibuilder'
+    )