Skip to content
Integration

ci: add Go/CGO to vcpkg and Windows-on-ARM tests (#4728) #6189

Sign in to view logs

ci: add Go/CGO to vcpkg and Windows-on-ARM tests (#4728)

ci: add Go/CGO to vcpkg and Windows-on-ARM tests (#4728) #6189

Workflow file for this run

.github/workflows/integration.yml at bb2be2d
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Integration
on:
pull_request:
branches:
- main
paths:
- "c/**"
- "ci/**"
- "go/**"
- "python/**"
- "compose.yaml"
- ".github/workflows/integration.yml"
push:
branches-ignore:
- 'dependabot/**'
paths:
- "c/**"
- "ci/**"
- "go/**"
- "python/**"
- "compose.yaml"
- ".github/workflows/integration.yml"
concurrency:
group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
cancel-in-progress: true
permissions:
contents: read
env:
# Increment this to reset cache manually
CACHE_NUMBER: "0"
defaults:
run:
shell: bash -l -eo pipefail {0}
jobs:
duckdb:
name: "DuckDB Integration Tests"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Get Date
id: get-date
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
with:
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
run: |
./ci/scripts/remamba.sh install -c conda-forge \
--file ci/conda_env_cpp.txt
- name: Work around ASAN issue (GH-1617)
run: |
sudo sysctl vm.mmap_rnd_bits=28
- name: Build DuckDB Integration Tests
env:
BUILD_ALL: "0"
BUILD_DRIVER_MANAGER: "1"
BUILD_INTEGRATION_DUCKDB: "1"
ADBC_USE_ASAN: "OFF"
ADBC_USE_UBSAN: "OFF"
# Make seems to have issues with the nested DuckDB build
ADBC_CMAKE_ARGS: "-GNinja"
run: |
./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
- name: Run DuckDB Integration Tests
env:
BUILD_ALL: "0"
BUILD_INTEGRATION_DUCKDB: "1"
run: |
./ci/scripts/cpp_test.sh "$(pwd)/build"
flightsql:
name: "FlightSQL Integration Tests (Dremio, SQLite, and GizmoSQL)"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Get required Go version
run: |
(. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV
- name: Get Date
id: get-date
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
with:
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
run: |
./ci/scripts/remamba.sh install -c conda-forge \
--file ci/conda_env_cpp.txt \
--file ci/conda_env_python.txt
pip install pytest-error-for-skips
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version: "${{ env.GO_VERSION }}"
check-latest: true
cache: true
cache-dependency-path: go/adbc/go.sum
- name: Work around ASAN issue (GH-1617)
run: |
sudo sysctl vm.mmap_rnd_bits=28
- name: Start SQLite server, Dremio, and GizmoSQL
run: |
docker compose up --detach flightsql-test flightsql-sqlite-test dremio dremio-init gizmosql-test
pip install python-dotenv[cli]
python -m dotenv -f .env list --format simple | tee -a $GITHUB_ENV
- name: Build FlightSQL Driver
env:
BUILD_ALL: "0"
BUILD_DRIVER_FLIGHTSQL: "1"
BUILD_DRIVER_MANAGER: "1"
run: |
./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
- name: Test FlightSQL Driver against Dremio and SQLite
env:
BUILD_ALL: "0"
BUILD_DRIVER_FLIGHTSQL: "1"
run: |
./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
./ci/scripts/cpp_test.sh "$(pwd)/build"
- name: Test Python Flight SQL driver against Dremio and GizmoSQL
env:
BUILD_ALL: "0"
BUILD_DRIVER_FLIGHTSQL: "1"
PYTEST_ADDOPTS: "--error-for-skips"
run: |
./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build"
- name: Stop SQLite server, Dremio, and GizmoSQL
run: |
docker compose down
postgresql-build:
name: "PostgreSQL Build"
runs-on: ubuntu-latest
steps:
- name: Free up disk space
run: |
# Preinstalled tools use a lot of disk space, free up some space
# https://github.com/actions/runner-images/issues/2840
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Get Date
id: get-date
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
with:
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
run: |
./ci/scripts/remamba.sh install -c conda-forge \
--file ci/conda_env_cpp.txt \
--file ci/conda_env_python.txt
pip install pytest-error-for-skips
- name: Work around ASAN issue (GH-1617)
run: |
sudo sysctl vm.mmap_rnd_bits=28
- name: Build PostgreSQL Driver
env:
BUILD_ALL: "OFF"
BUILD_DRIVER_MANAGER: "ON"
BUILD_DRIVER_POSTGRESQL: "ON"
run: |
./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
# Artifact actions do not preserve symlinks, so archive the build tree and
# the generated Python extension/library files ourselves.
- name: Archive PostgreSQL build
run: |
tar --create --gzip --file postgresql-build.tgz \
build \
python/adbc_driver_manager \
python/adbc_driver_postgresql
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: postgresql-build
retention-days: 3
path: postgresql-build.tgz
postgresql-test:
name: "PostgreSQL (14-18)"
needs: postgresql-build
runs-on: ubuntu-latest
steps:
- name: Free up disk space
run: |
# Preinstalled tools use a lot of disk space, free up some space
# https://github.com/actions/runner-images/issues/2840
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Get Date
id: get-date
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
with:
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
run: |
./ci/scripts/remamba.sh install -c conda-forge \
--file ci/conda_env_cpp.txt \
--file ci/conda_env_python.txt
pip install pytest-error-for-skips
- name: Work around ASAN issue (GH-1617)
run: |
sudo sysctl vm.mmap_rnd_bits=28
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: postgresql-build
- name: Extract PostgreSQL build
run: |
tar --extract --gzip --file postgresql-build.tgz
echo "PYTHONPATH=$(pwd)/python/adbc_driver_manager:$(pwd)/python/adbc_driver_postgresql" >> "$GITHUB_ENV"
- name: Run PostgreSQL 14-18 tests
env:
ADBC_POSTGRESQL_TEST_URI: postgresql://localhost:5432/postgres?user=postgres&password=password
ADBC_USE_ASAN: "ON"
ADBC_USE_UBSAN: "ON"
BUILD_ALL: "0"
BUILD_DRIVER_POSTGRESQL: "1"
run: |
for postgres_version in 14 15 16 17 18; do
echo "::group::PostgreSQL $postgres_version"
export POSTGRES_VERSION="$postgres_version"
docker compose up --wait --detach postgres-test
./ci/scripts/cpp_test.sh "$(pwd)/build"
env PYTEST_ADDOPTS=--error-for-skips \
./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build"
if [ "$postgres_version" = "18" ]; then
uv --project=./c/driver/postgresql/validation/ run \
ci/scripts/validate.sh "$(pwd)" --vendor postgresql
fi
docker compose stop postgres-test
docker compose rm --force --stop --volumes postgres-test
echo "::endgroup::"
done
- name: Capture PostgreSQL logs
if: failure()
run: |
docker compose logs postgres-test
postgresql-vendor-test:
name: "PostgreSQL (${{ matrix.name }})"
needs: postgresql-build
runs-on: ubuntu-latest
continue-on-error: ${{ matrix.experimental }}
strategy:
fail-fast: false
max-parallel: 2
matrix:
include:
- name: AlloyDB Omni
service: alloydb-omni-test
vendor: alloydb-omni
uri: postgresql://localhost:5440/postgres?user=postgres&password=password
experimental: false
- name: Citus
service: citus-test
vendor: citus
uri: postgresql://localhost:5434/postgres?user=postgres&password=password
experimental: false
- name: CockroachDB
service: cockroachdb-test
vendor: cockroachdb
uri: postgresql://root@localhost:5435/postgres?sslmode=disable
experimental: false
- name: CrateDB
service: cratedb-test
vendor: cratedb
uri: postgresql://crate@localhost:5436/doc?sslmode=disable
experimental: true
- name: ParadeDB
service: paradedb-test
vendor: paradedb
uri: postgresql://localhost:5437/postgres?user=postgres&password=password
experimental: false
- name: TimescaleDB
service: timescaledb-test
vendor: timescaledb
uri: postgresql://localhost:5438/postgres?user=postgres&password=password
experimental: false
- name: YugabyteDB
service: yugabytedb-test
vendor: yugabytedb
uri: postgresql://localhost:5439/yugabyte?user=yugabyte&password=yugabyte
experimental: false
# Currently fails, but could be supported in the future.
# https://github.com/apache/arrow-adbc/issues/1755
# - name: CedarDB
# service: cedardb-test
# vendor: cedardb
# uri: postgresql://localhost:5433/postgres?user=postgres&password=CedarDB2026%21
# experimental: true
steps:
- name: Free up disk space
run: |
# Preinstalled tools use a lot of disk space, free up some space
# https://github.com/actions/runner-images/issues/2840
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Get Date
id: get-date
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
with:
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
run: |
./ci/scripts/remamba.sh install -c conda-forge \
--file ci/conda_env_cpp.txt \
--file ci/conda_env_python.txt
pip install pytest-error-for-skips
- name: Work around ASAN issue (GH-1617)
run: |
sudo sysctl vm.mmap_rnd_bits=28
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: postgresql-build
- name: Extract PostgreSQL build
run: |
tar --extract --gzip --file postgresql-build.tgz
echo "PYTHONPATH=$(pwd)/python/adbc_driver_manager:$(pwd)/python/adbc_driver_postgresql" >> "$GITHUB_ENV"
- name: Start ${{ matrix.name }}
run: |
docker compose up --wait --detach "${{ matrix.service }}"
- name: Run validation suite
env:
ADBC_POSTGRESQL_TEST_URI: ${{ matrix.uri }}
ADBC_USE_ASAN: "ON"
ADBC_USE_UBSAN: "ON"
BUILD_ALL: "0"
BUILD_DRIVER_POSTGRESQL: "1"
run: |
uv --project=./c/driver/postgresql/validation/ run \
ci/scripts/validate.sh "$(pwd)" --vendor "${{ matrix.vendor }}"
- name: Capture ${{ matrix.name }} logs
if: failure()
run: |
docker compose logs "${{ matrix.service }}"
flightsql_interop:
name: "FlightSQL C# Interop"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Get required Go version
run: |
(. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version: "${{ env.GO_VERSION }}"
check-latest: true
cache: true
cache-dependency-path: go/adbc/go.sum
- name: Build ADBC Driver
working-directory: go/adbc/pkg
run: |
make libadbc_driver_flightsql.so
- name: Start Test Servers
run: |
docker compose up --wait --detach spiceai-test
- name: Test Driver against Spice.ai OSS
env:
FLIGHTSQL_INTEROP_TEST_CONFIG_FILE: "../../../../../csharp/configs/flightsql-spiceai.json"
run: |
dotnet test ./csharp/test/Drivers/Interop/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.Interop.FlightSql.csproj
- name: Stop Test Servers
run: |
docker compose down