chore(dev/release): build BigQuery wheels

Fixes apache#2110.

c/driver/bigquery/bigquery_test.cc

@@ -15,15 +15,18 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>
+#include <cstring>
+#include <random>
+#include <thread>
+
 #include <arrow-adbc/adbc.h>
 #include <gmock/gmock-matchers.h>
 #include <gtest/gtest-matchers.h>
 #include <gtest/gtest-param-test.h>
 #include <gtest/gtest.h>
 #include <nanoarrow/nanoarrow.h>
-#include <algorithm>
-#include <cstring>
-#include <random>
+
 #include "validation/adbc_validation.h"
 #include "validation/adbc_validation_util.h"
 
@@ -120,7 +123,9 @@ class BigQueryQuirks : public adbc_validation::DriverQuirks {
     create += "` (int64s INT, strings TEXT)";
     CHECK_OK(AdbcStatementSetSqlQuery(&statement.value, create.c_str(), error));
     CHECK_OK(AdbcStatementExecuteQuery(&statement.value, nullptr, nullptr, error));
-    sleep(5);
+    // XXX: is there a better way to wait for BigQuery? (Why does 'CREATE
+    // TABLE' not wait for commit?)
+    std::this_thread::sleep_for(std::chrono::seconds(5));
 
     std::string insert = "INSERT INTO `ADBC_TESTING.";
     insert += name;

ci/scripts/cpp_build.ps1

@@ -24,10 +24,11 @@ $InstallDir = if ($Args[2] -ne $null) { $Args[2] } else { Join-Path $BuildDir "l
 
 $BuildAll = $env:BUILD_ALL -ne "0"
 $BuildDriverManager = ($BuildAll -and (-not ($env:BUILD_DRIVER_MANAGER -eq "0"))) -or ($env:BUILD_DRIVER_MANAGER -eq "1")
+$BuildDriverBigQuery = ($BuildAll -and (-not ($env:BUILD_DRIVER_BIGQUERY -eq "0"))) -or ($env:BUILD_DRIVER_BIGQUERY -eq "1")
 $BuildDriverFlightSql = ($BuildAll -and (-not ($env:BUILD_DRIVER_FLIGHTSQL -eq "0"))) -or ($env:BUILD_DRIVER_FLIGHTSQL -eq "1")
 $BuildDriverPostgreSQL = ($BuildAll -and (-not ($env:BUILD_DRIVER_POSTGRESQL -eq "0"))) -or ($env:BUILD_DRIVER_POSTGRESQL -eq "1")
-$BuildDriverSqlite = ($BuildAll -and (-not ($env:BUILD_DRIVER_SQLITE -eq "0"))) -or ($env:BUILD_DRIVER_SQLITE -eq "1")
 $BuildDriverSnowflake = ($BuildAll -and (-not ($env:BUILD_DRIVER_SNOWFLAKE -eq "0"))) -or ($env:BUILD_DRIVER_SNOWFLAKE -eq "1")
+$BuildDriverSqlite = ($BuildAll -and (-not ($env:BUILD_DRIVER_SQLITE -eq "0"))) -or ($env:BUILD_DRIVER_SQLITE -eq "1")
 
 function Build-Subproject {
     New-Item -ItemType Directory -Force -Path $BuildDir | Out-Null
@@ -40,10 +41,11 @@ function Build-Subproject {
       -DADBC_BUILD_STATIC=OFF `
       -DADBC_BUILD_TESTS=ON `
       -DADBC_DRIVER_MANAGER="$($BuildDriverManager)" `
+      -DADBC_DRIVER_BIGQUERY="$($BuildDriverBigQuery)" `
       -DADBC_DRIVER_FLIGHTSQL="$($BuildDriverFlightSql)" `
       -DADBC_DRIVER_POSTGRESQL="$($BuildDriverPostgreSQL)" `
-      -DADBC_DRIVER_SQLITE="$($BuildDriverSqlite)" `
       -DADBC_DRIVER_SNOWFLAKE="$($BuildDriverSnowflake)" `
+      -DADBC_DRIVER_SQLITE="$($BuildDriverSqlite)" `
       -DCMAKE_BUILD_TYPE=Release `
       -DCMAKE_INSTALL_PREFIX="$($InstallDir)" `
       -DCMAKE_VERBOSE_MAKEFILE=ON

ci/scripts/cpp_build.sh

@@ -57,12 +57,12 @@ build_subproject() {
           -DADBC_BUILD_SHARED="${ADBC_BUILD_SHARED}" \
           -DADBC_BUILD_STATIC="${ADBC_BUILD_STATIC}" \
           -DADBC_BUILD_TESTS="${ADBC_BUILD_TESTS}" \
+          -DADBC_DRIVER_BIGQUERY="${BUILD_DRIVER_BIGQUERY}" \
+          -DADBC_DRIVER_FLIGHTSQL="${BUILD_DRIVER_FLIGHTSQL}" \
           -DADBC_DRIVER_MANAGER="${BUILD_DRIVER_MANAGER}" \
           -DADBC_DRIVER_POSTGRESQL="${BUILD_DRIVER_POSTGRESQL}" \
-          -DADBC_DRIVER_SQLITE="${BUILD_DRIVER_SQLITE}" \
-          -DADBC_DRIVER_FLIGHTSQL="${BUILD_DRIVER_FLIGHTSQL}" \
           -DADBC_DRIVER_SNOWFLAKE="${BUILD_DRIVER_SNOWFLAKE}" \
-          -DADBC_DRIVER_BIGQUERY="${BUILD_DRIVER_BIGQUERY}" \
+          -DADBC_DRIVER_SQLITE="${BUILD_DRIVER_SQLITE}" \
           -DADBC_INTEGRATION_DUCKDB="${BUILD_INTEGRATION_DUCKDB}" \
           -DADBC_USE_ASAN="${ADBC_USE_ASAN}" \
           -DADBC_USE_UBSAN="${ADBC_USE_UBSAN}" \

ci/scripts/cpp_test.ps1

@@ -23,10 +23,11 @@ $InstallDir = if ($Args[1] -ne $null) { $Args[1] } else { Join-Path $BuildDir "l
 
 $BuildAll = $env:BUILD_ALL -ne "0"
 $BuildDriverManager = ($BuildAll -and (-not ($env:BUILD_DRIVER_MANAGER -eq "0"))) -or ($env:BUILD_DRIVER_MANAGER -eq "1")
+$BuildDriverBigQuery = ($BuildAll -and (-not ($env:BUILD_DRIVER_BIGQUERY -eq "0"))) -or ($env:BUILD_DRIVER_BIGQUERY -eq "1")
 $BuildDriverFlightSql = ($BuildAll -and (-not ($env:BUILD_DRIVER_FLIGHTSQL -eq "0"))) -or ($env:BUILD_DRIVER_FLIGHTSQL -eq "1")
 $BuildDriverPostgreSQL = ($BuildAll -and (-not ($env:BUILD_DRIVER_POSTGRESQL -eq "0"))) -or ($env:BUILD_DRIVER_POSTGRESQL -eq "1")
-$BuildDriverSqlite = ($BuildAll -and (-not ($env:BUILD_DRIVER_SQLITE -eq "0"))) -or ($env:BUILD_DRIVER_SQLITE -eq "1")
 $BuildDriverSnowflake = ($BuildAll -and (-not ($env:BUILD_DRIVER_SNOWFLAKE -eq "0"))) -or ($env:BUILD_DRIVER_SNOWFLAKE -eq "1")
+$BuildDriverSqlite = ($BuildAll -and (-not ($env:BUILD_DRIVER_SQLITE -eq "0"))) -or ($env:BUILD_DRIVER_SQLITE -eq "1")
 
 $env:LD_LIBRARY_PATH += ":$($InstallDir)"
 $env:LD_LIBRARY_PATH += ":$($InstallDir)/bin"
@@ -43,18 +44,21 @@ function Test-Project {
     if ($BuildDriverManager) {
         $labels += "|driver-manager"
     }
+    if ($BuildDriverFlightSql) {
+        $labels += "|driver-bigquery"
+    }
     if ($BuildDriverFlightSql) {
         $labels += "|driver-flightsql"
     }
     if ($BuildDriverPostgreSQL) {
         $labels += "|driver-postgresql"
     }
-    if ($BuildDriverSqlite) {
-        $labels += "|driver-sqlite"
-    }
     if ($BuildDriverSnowflake) {
         $labels += "|driver-snowflake"
     }
+    if ($BuildDriverSqlite) {
+        $labels += "|driver-sqlite"
+    }
 
     ctest --output-on-failure --no-tests=error -L "$($labels)"
     if (-not $?) { exit 1 }

ci/scripts/cpp_test.sh

@@ -33,6 +33,9 @@ test_project() {
     pushd "${build_dir}/"
 
     local labels=""
+    if [[ "${BUILD_DRIVER_BIGQUERY}" -gt 0 ]]; then
+       labels="${labels}|driver-bigquery"
+    fi
     if [[ "${BUILD_DRIVER_FLIGHTSQL}" -gt 0 ]]; then
        labels="${labels}|driver-flightsql"
     fi
@@ -42,14 +45,11 @@ test_project() {
     if [[ "${BUILD_DRIVER_POSTGRESQL}" -gt 0 ]]; then
        labels="${labels}|driver-postgresql"
     fi
-    if [[ "${BUILD_DRIVER_SQLITE}" -gt 0 ]]; then
-       labels="${labels}|driver-sqlite"
-    fi
     if [[ "${BUILD_DRIVER_SNOWFLAKE}" -gt 0 ]]; then
        labels="${labels}|driver-snowflake"
     fi
-    if [[ "${BUILD_DRIVER_BIGQUERY}" -gt 0 ]]; then
-       labels="${labels}|driver-bigquery"
+    if [[ "${BUILD_DRIVER_SQLITE}" -gt 0 ]]; then
+       labels="${labels}|driver-sqlite"
     fi
     if [[ "${BUILD_INTEGRATION_DUCKDB}" -gt 0 ]]; then
        labels="${labels}|integration-duckdb"

ci/scripts/python_build.ps1

@@ -22,17 +22,19 @@ $SourceDir = $Args[0]
 $BuildDir = $Args[1]
 
 $BuildAll = $env:BUILD_ALL -ne "0"
+$BuildDriverBigQuery = ($BuildAll -and (-not ($env:BUILD_DRIVER_BIGQUERY -eq "0"))) -or ($env:BUILD_DRIVER_BIGQUERY -eq "1")
 $BuildDriverFlightSql = ($BuildAll -and (-not ($env:BUILD_DRIVER_FLIGHTSQL -eq "0"))) -or ($env:BUILD_DRIVER_FLIGHTSQL -eq "1")
 $BuildDriverManager = ($BuildAll -and (-not ($env:BUILD_DRIVER_MANAGER -eq "0"))) -or ($env:BUILD_DRIVER_MANAGER -eq "1")
 $BuildDriverPostgreSQL = ($BuildAll -and (-not ($env:BUILD_DRIVER_POSTGRESQL -eq "0"))) -or ($env:BUILD_DRIVER_POSTGRESQL -eq "1")
-$BuildDriverSqlite = ($BuildAll -and (-not ($env:BUILD_DRIVER_SQLITE -eq "0"))) -or ($env:BUILD_DRIVER_SQLITE -eq "1")
 $BuildDriverSnowflake = ($BuildAll -and (-not ($env:BUILD_DRIVER_SNOWFLAKE -eq "0"))) -or ($env:BUILD_DRIVER_SNOWFLAKE -eq "1")
+$BuildDriverSqlite = ($BuildAll -and (-not ($env:BUILD_DRIVER_SQLITE -eq "0"))) -or ($env:BUILD_DRIVER_SQLITE -eq "1")
 
 cmake -S "$($SourceDir)\c" -B $BuildDir `
     -DADBC_DRIVER_MANAGER=$BuildDriverManager `
+    -DADBC_DRIVER_BIGQUERY=$BuildDriverFlightSql `
     -DADBC_DRIVER_FLIGHTSQL=$BuildDriverFlightSql `
     -DADBC_DRIVER_POSTGRESQL=$BuildDriverPostgreSQL `
-    -DADBC_DRIVER_SQLITE=$BuildDriverSqlite `
     -DADBC_DRIVER_SNOWFLAKE=$BuildDriverSnowflake `
+    -DADBC_DRIVER_SQLITE=$BuildDriverSqlite `
     -DADBC_BUILD_PYTHON=ON
 cmake --build $BuildDir --target python

ci/scripts/python_build.sh

@@ -19,6 +19,7 @@
 set -e
 
 : ${BUILD_ALL:=1}
+: ${BUILD_DRIVER_BIGQUERY:=${BUILD_ALL}}
 : ${BUILD_DRIVER_FLIGHTSQL:=${BUILD_ALL}}
 : ${BUILD_DRIVER_MANAGER:=${BUILD_ALL}}
 : ${BUILD_DRIVER_POSTGRESQL:=${BUILD_ALL}}
@@ -43,6 +44,7 @@ main() {
           -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \
           -DADBC_USE_ASAN="${ADBC_USE_ASAN}" \
           -DADBC_USE_UBSAN="${ADBC_USE_UBSAN}" \
+          -DADBC_DRIVER_BIGQUERY=${BUILD_DRIVER_MANAGER} \
           -DADBC_DRIVER_MANAGER=${BUILD_DRIVER_MANAGER} \
           -DADBC_DRIVER_FLIGHTSQL=${BUILD_DRIVER_FLIGHTSQL} \
           -DADBC_DRIVER_POSTGRESQL=${BUILD_DRIVER_POSTGRESQL} \

ci/scripts/python_test.ps1

@@ -24,10 +24,11 @@ $InstallDir = if ($Args[2] -ne $null) { $Args[2] } else { Join-Path $BuildDir "l
 
 $BuildAll = $env:BUILD_ALL -ne "0"
 $BuildDriverManager = ($BuildAll -and (-not ($env:BUILD_DRIVER_MANAGER -eq "0"))) -or ($env:BUILD_DRIVER_MANAGER -eq "1")
+$BuildDriverBigQuery = ($BuildAll -and (-not ($env:BUILD_DRIVER_BIGQUERY -eq "0"))) -or ($env:BUILD_DRIVER_BIGQUERY -eq "1")
 $BuildDriverFlightSql = ($BuildAll -and (-not ($env:BUILD_DRIVER_FLIGHTSQL -eq "0"))) -or ($env:BUILD_DRIVER_FLIGHTSQL -eq "1")
 $BuildDriverPostgreSQL = ($BuildAll -and (-not ($env:BUILD_DRIVER_POSTGRESQL -eq "0"))) -or ($env:BUILD_DRIVER_POSTGRESQL -eq "1")
-$BuildDriverSqlite = ($BuildAll -and (-not ($env:BUILD_DRIVER_SQLITE -eq "0"))) -or ($env:BUILD_DRIVER_SQLITE -eq "1")
 $BuildDriverSnowflake = ($BuildAll -and (-not ($env:BUILD_DRIVER_SNOWFLAKE -eq "0"))) -or ($env:BUILD_DRIVER_SNOWFLAKE -eq "1")
+$BuildDriverSqlite = ($BuildAll -and (-not ($env:BUILD_DRIVER_SQLITE -eq "0"))) -or ($env:BUILD_DRIVER_SQLITE -eq "1")
 
 function Build-Subproject {
     $Subproject = $Args[0]
@@ -55,6 +56,9 @@ if ($BuildDriverManager) {
     $env:PATH += ";$($SqliteDir)"
     Build-Subproject adbc_driver_manager
 }
+if ($BuildDriverBigQuery) {
+    Build-Subproject adbc_driver_bigquery
+}
 if ($BuildDriverFlightSql) {
     Build-Subproject adbc_driver_flightsql
 }

ci/scripts/python_test.sh

@@ -21,6 +21,7 @@ set -e
 : ${ADBC_USE_ASAN:=OFF}
 : ${ADBC_USE_UBSAN:=OFF}
 : ${BUILD_ALL:=1}
+: ${BUILD_DRIVER_BIGQUERY:=${BUILD_ALL}}
 : ${BUILD_DRIVER_FLIGHTSQL:=${BUILD_ALL}}
 : ${BUILD_DRIVER_MANAGER:=${BUILD_ALL}}
 : ${BUILD_DRIVER_POSTGRESQL:=${BUILD_ALL}}
@@ -72,6 +73,10 @@ main() {
         install_dir="${build_dir}/local"
     fi
 
+    if [[ "${BUILD_DRIVER_BIGQUERY}" -gt 0 ]]; then
+        test_subproject "${source_dir}" "${install_dir}" adbc_driver_bigquery
+    fi
+
     if [[ "${BUILD_DRIVER_FLIGHTSQL}" -gt 0 ]]; then
         test_subproject "${source_dir}" "${install_dir}" adbc_driver_flightsql
     fi

ci/scripts/python_util.sh

@@ -19,7 +19,7 @@
 
 set -ex
 
-COMPONENTS="adbc_driver_manager adbc_driver_flightsql adbc_driver_postgresql adbc_driver_sqlite adbc_driver_snowflake"
+COMPONENTS="adbc_driver_bigquery adbc_driver_manager adbc_driver_flightsql adbc_driver_postgresql adbc_driver_sqlite adbc_driver_snowflake"
 
 function build_drivers {
     local -r source_dir="$1"
@@ -36,13 +36,15 @@ function build_drivers {
     export VCPKG_OVERLAY_TRIPLETS="${source_dir}/ci/vcpkg/triplets/"
 
     if [[ $(uname) == "Linux" ]]; then
+        export ADBC_BIGQUERY_LIBRARY=${build_dir}/lib/libadbc_driver_bigquery.so
         export ADBC_FLIGHTSQL_LIBRARY=${build_dir}/lib/libadbc_driver_flightsql.so
         export ADBC_POSTGRESQL_LIBRARY=${build_dir}/lib/libadbc_driver_postgresql.so
         export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.so
         export ADBC_SNOWFLAKE_LIBRARY=${build_dir}/lib/libadbc_driver_snowflake.so
         export VCPKG_DEFAULT_TRIPLET="${VCPKG_ARCH}-linux-static-release"
         export CMAKE_ARGUMENTS=""
     else # macOS
+        export ADBC_BIGQUERY_LIBRARY=${build_dir}/lib/libadbc_driver_bigquery.dylib
         export ADBC_FLIGHTSQL_LIBRARY=${build_dir}/lib/libadbc_driver_flightsql.dylib
         export ADBC_POSTGRESQL_LIBRARY=${build_dir}/lib/libadbc_driver_postgresql.dylib
         export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.dylib
@@ -91,6 +93,7 @@ function build_drivers {
         ${CMAKE_ARGUMENTS} \
         -DVCPKG_OVERLAY_TRIPLETS="${VCPKG_OVERLAY_TRIPLETS}" \
         -DVCPKG_TARGET_TRIPLET="${VCPKG_DEFAULT_TRIPLET}" \
+        -DADBC_DRIVER_BIGQUERY=ON \
         -DADBC_DRIVER_FLIGHTSQL=ON \
         -DADBC_DRIVER_POSTGRESQL=ON \
         -DADBC_DRIVER_SQLITE=ON \

ci/scripts/python_wheel_unix_build.sh

@@ -79,6 +79,7 @@ echo "=== Building C/C++ driver components ==="
 build_drivers "${source_dir}" "${build_dir}"
 
 # Check that we don't expose any unwanted symbols
+check_visibility $ADBC_BIGQUERY_LIBRARY
 check_visibility $ADBC_FLIGHTSQL_LIBRARY
 check_visibility $ADBC_POSTGRESQL_LIBRARY
 check_visibility $ADBC_SQLITE_LIBRARY

ci/scripts/python_wheel_windows_build.bat

@@ -47,15 +47,17 @@ cmake ^
       -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%/scripts/buildsystems/vcpkg.cmake ^
       -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
       -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^
-      -DADBC_DRIVER_POSTGRESQL=ON ^
-      -DADBC_DRIVER_SQLITE=ON ^
+      -DADBC_DRIVER_BIGQUERY=ON ^
       -DADBC_DRIVER_FLIGHTSQL=ON ^
       -DADBC_DRIVER_MANAGER=ON ^
+      -DADBC_DRIVER_POSTGRESQL=ON ^
       -DADBC_DRIVER_SNOWFLAKE=ON ^
+      -DADBC_DRIVER_SQLITE=ON ^
       %source_dir%\c || exit /B 1
 
 cmake --build . --config %CMAKE_BUILD_TYPE% --target install --verbose -j || exit /B 1
 
+set ADBC_BIGQUERY_LIBRARY=%build_dir%\bin\adbc_driver_bigquery.dll
 set ADBC_FLIGHTSQL_LIBRARY=%build_dir%\bin\adbc_driver_flightsql.dll
 set ADBC_POSTGRESQL_LIBRARY=%build_dir%\bin\adbc_driver_postgresql.dll
 set ADBC_SQLITE_LIBRARY=%build_dir%\bin\adbc_driver_sqlite.dll
@@ -70,7 +72,7 @@ python -m pip install --upgrade pip delvewheel wheel || exit /B 1
 
 FOR /F %%i IN ('python -c "import sysconfig; print(sysconfig.get_platform())"') DO set PLAT_NAME=%%i
 
-FOR %%c IN (adbc_driver_manager adbc_driver_flightsql adbc_driver_postgresql adbc_driver_sqlite adbc_driver_snowflake) DO (
+FOR %%c IN (adbc_driver_bigquery adbc_driver_manager adbc_driver_flightsql adbc_driver_postgresql adbc_driver_sqlite adbc_driver_snowflake) DO (
     pushd %source_dir%\python\%%c
 
     echo "=== (%PYTHON_VERSION%) Checking %%c version ==="

ci/scripts/python_wheel_windows_test.bat

@@ -21,7 +21,7 @@ set source_dir=%1
 
 echo "=== (%PYTHON_VERSION%) Installing wheels ==="
 
-FOR %%c IN (adbc_driver_manager adbc_driver_flightsql adbc_driver_postgresql adbc_driver_sqlite adbc_driver_snowflake) DO (
+FOR %%c IN (adbc_driver_bigquery adbc_driver_manager adbc_driver_flightsql adbc_driver_postgresql adbc_driver_sqlite adbc_driver_snowflake) DO (
     FOR %%w IN (%source_dir%\python\%%c\dist\*.whl) DO (
         pip install --no-deps --force-reinstall %%w || exit /B 1
     )
@@ -31,7 +31,7 @@ pip install importlib-resources pytest pyarrow pandas protobuf
 
 echo "=== (%PYTHON_VERSION%) Testing wheels ==="
 
-FOR %%c IN (adbc_driver_manager adbc_driver_flightsql adbc_driver_postgresql adbc_driver_sqlite adbc_driver_snowflake) DO (
+FOR %%c IN (adbc_driver_bigquery adbc_driver_manager adbc_driver_flightsql adbc_driver_postgresql adbc_driver_sqlite adbc_driver_snowflake) DO (
     echo "=== Testing %%c ==="
     python -c "import %%c" || exit /B 1
     python -c "import %%c.dbapi" || exit /B 1

python/adbc_driver_bigquery/adbc_driver_bigquery/__init__.py

@@ -231,7 +231,6 @@ def _driver_path() -> str:
     # The filename is always the same regardless of platform
     entrypoint = root.joinpath(f"lib{driver}.so")
     if entrypoint.is_file():
-        print(str(entrypoint))
         return str(entrypoint)
 
     # Search sys.prefix + '/lib' (Unix, Conda on Unix)

python/adbc_driver_bigquery/adbc_driver_bigquery/_static_version.py

@@ -20,7 +20,7 @@
 
 # This file is part of 'miniver': https://github.com/jbweston/miniver
 
-version = "1.0.0dev"
+version = "1.2.0dev"
 
 # These values are only set if the distribution was created with 'git archive'
 # NOTE: must add an export-subst to .gitattributes!

python/adbc_driver_bigquery/tests/test_lowlevel.py

@@ -44,8 +44,9 @@ def test_load_driver():
     with pytest.raises(
         adbc_driver_manager.ProgrammingError, match="ProjectID is empty"
     ):
-        with adbc_driver_bigquery.connect(""):
-            pass
+        with adbc_driver_bigquery.connect("") as db:
+            with adbc_driver_manager.AdbcConnection(db):
+                pass
 
 
 def test_query_trivial(bigquery):