From a7cba6983e876228e529ca1871b7bb21ca86f325 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 5 Sep 2024 14:51:10 -0500 Subject: [PATCH 01/10] avoid desc where big endian might be used --- ci/docker/alpine.dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/docker/alpine.dockerfile b/ci/docker/alpine.dockerfile index e4d2c58e9..d8a3cdbc4 100644 --- a/ci/docker/alpine.dockerfile +++ b/ci/docker/alpine.dockerfile @@ -33,9 +33,8 @@ ENV NANOARROW_PYTHON_VENV "/venv" # For R. Note that arrow is not installed (takes too long). RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars -RUN R -e 'install.packages("desc", repos = "https://cloud.r-project.org")' && mkdir /tmp/rdeps COPY r/DESCRIPTION /tmp/rdeps -RUN R -e 'install.packages(setdiff(desc::desc("/tmp/rdeps")$get_deps()$package, "arrow"), repos = "https://cloud.r-project.org")' +RUN R -e 'gsub("\\(.*?\\)", "", read.dcf("../arrow-nanoarrow/r/DESCRIPTION")[1, "Suggests"]) |> strsplit("[^A-Za-z0-9.]+") |> unlist(use.names = FALSE) |> setdiff("arrow") |> install.packages(repos = "https://cloud.r-project.org")' RUN rm -f ~/.R/Makevars ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow From f18f293edbad8043eae648af46abfc2467da86be Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 5 Sep 2024 14:55:57 -0500 Subject: [PATCH 02/10] maybe fix path --- ci/docker/alpine.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/alpine.dockerfile b/ci/docker/alpine.dockerfile index d8a3cdbc4..45d33461a 100644 --- a/ci/docker/alpine.dockerfile +++ b/ci/docker/alpine.dockerfile @@ -34,7 +34,7 @@ ENV NANOARROW_PYTHON_VENV "/venv" # For R. Note that arrow is not installed (takes too long). RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars COPY r/DESCRIPTION /tmp/rdeps -RUN R -e 'gsub("\\(.*?\\)", "", read.dcf("../arrow-nanoarrow/r/DESCRIPTION")[1, "Suggests"]) |> strsplit("[^A-Za-z0-9.]+") |> unlist(use.names = FALSE) |> setdiff("arrow") |> install.packages(repos = "https://cloud.r-project.org")' +RUN R -e 'gsub("\\(.*?\\)", "", read.dcf("/tmp/rdeps/DESCRIPTION")[1, "Suggests"]) |> strsplit("[^A-Za-z0-9.]+") |> unlist(use.names = FALSE) |> setdiff("arrow") |> install.packages(repos = "https://cloud.r-project.org")' RUN rm -f ~/.R/Makevars ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow From f5aaa1e9a6d5f23ff767ce60fcd164935c8ac9f0 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 5 Sep 2024 15:09:12 -0500 Subject: [PATCH 03/10] maybe fix path again --- ci/docker/alpine.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/alpine.dockerfile b/ci/docker/alpine.dockerfile index 45d33461a..1c9d51374 100644 --- a/ci/docker/alpine.dockerfile +++ b/ci/docker/alpine.dockerfile @@ -34,7 +34,7 @@ ENV NANOARROW_PYTHON_VENV "/venv" # For R. Note that arrow is not installed (takes too long). RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars COPY r/DESCRIPTION /tmp/rdeps -RUN R -e 'gsub("\\(.*?\\)", "", read.dcf("/tmp/rdeps/DESCRIPTION")[1, "Suggests"]) |> strsplit("[^A-Za-z0-9.]+") |> unlist(use.names = FALSE) |> setdiff("arrow") |> install.packages(repos = "https://cloud.r-project.org")' +RUN R -e 'gsub("\\(.*?\\)", "", read.dcf("/tmp/rdeps")[1, "Suggests"]) |> strsplit("[^A-Za-z0-9.]+") |> unlist(use.names = FALSE) |> setdiff("arrow") |> install.packages(repos = "https://cloud.r-project.org")' RUN rm -f ~/.R/Makevars ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow From f86b2b68735860a85d7582ab1e6a7ffdd532f0cb Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 5 Sep 2024 16:02:21 -0500 Subject: [PATCH 04/10] try to fix integration --- .github/workflows/docker-build.yaml | 1 - .github/workflows/integration.yaml | 2 +- docker-compose.yml | 8 ++------ 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index 4e9cea12c..fb60aa915 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -46,7 +46,6 @@ jobs: - { runs_on: "ubuntu-latest", platform: "fedora", arch: "amd64", service: "verify" } - { runs_on: "ubuntu-latest", platform: "archlinux", arch: "amd64", service: "verify" } - { runs_on: "ubuntu-latest", platform: "alpine", arch: "amd64", service: "verify" } - - { runs_on: "ubuntu-latest", arch: "amd64", platform: "integration", service: "integration" } - { runs_on: ["self-hosted", "arm"], platform: "ubuntu", arch: "arm64", service: "verify" } - { runs_on: ["self-hosted", "arm"], platform: "fedora", arch: "arm64", service: "verify" } diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 8125f2610..0d24a7b88 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -37,5 +37,5 @@ jobs: - name: Run integration tests run: | - echo "::group::Docker Pull" + echo "::group::Docker Build" docker compose run --rm -e GITHUB_ACTIONS integration diff --git a/docker-compose.yml b/docker-compose.yml index 96defb6ec..ca6291701 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -version: '3.5' - services: verify: image: ${REPO}:${NANOARROW_PLATFORM}-${NANOARROW_ARCH} @@ -48,12 +46,10 @@ services: command: "/bin/bash /build-docs.sh /nanoarrow" integration: - # Based on an arrow-dev repo image that is amd64 only - image: ${REPO}:integration-amd64 + # Don't cache here (building the image takes about the same amount of time + # as downloading on CI) build: context: . - cache_from: - - ${REPO}:integration-amd64 dockerfile: ci/docker/integration.dockerfile volumes: - ${NANOARROW_DOCKER_SOURCE_DIR}:/arrow-integration/nanoarrow From e629a8b98c2221797a5fef5e27d5b0b1bccebcf3 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 6 Sep 2024 09:59:33 -0500 Subject: [PATCH 05/10] change frequency of integration job --- .github/workflows/integration.yaml | 5 ++++- .github/workflows/python-wheels.yaml | 1 + .github/workflows/verify.yaml | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 0d24a7b88..966ac4b4f 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -21,10 +21,13 @@ on: push: branches: - main + - 'maint-**' pull_request: paths: - .github/workflows/integration.yaml - - src/nanoarrow/** + - docker-compose.yml + schedule: + - cron: '5 0 * * 0' permissions: contents: read diff --git a/.github/workflows/python-wheels.yaml b/.github/workflows/python-wheels.yaml index 3e8b723a6..acb9e0a63 100644 --- a/.github/workflows/python-wheels.yaml +++ b/.github/workflows/python-wheels.yaml @@ -31,6 +31,7 @@ on: push: branches: - main + - 'maint-**' workflow_dispatch: permissions: diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 0ff5b796f..b234f535a 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -18,6 +18,9 @@ name: Verification on: + push: + branches: + - 'maint-**' workflow_dispatch: inputs: version: From 14d24a31e2c62b85b76dc8e41c76c49cf3e5a5f7 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 6 Sep 2024 10:43:21 -0500 Subject: [PATCH 06/10] fix test for centos7 --- src/nanoarrow/ipc/writer_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nanoarrow/ipc/writer_test.cc b/src/nanoarrow/ipc/writer_test.cc index 5eed08092..5b06bc3b8 100644 --- a/src/nanoarrow/ipc/writer_test.cc +++ b/src/nanoarrow/ipc/writer_test.cc @@ -92,8 +92,8 @@ TEST(NanoarrowIpcWriter, OutputStreamFile) { TEST(NanoarrowIpcWriter, OutputStreamFileError) { nanoarrow::ipc::UniqueOutputStream stream; - EXPECT_EQ(ArrowIpcOutputStreamInitFile(stream.get(), nullptr, /*close_on_release=*/1), - EINVAL); + EXPECT_NE(ArrowIpcOutputStreamInitFile(stream.get(), nullptr, /*close_on_release=*/1), + NANOARROW_OK); auto phony_path = __FILE__ + std::string(".phony"); FILE* file_ptr = fopen(phony_path.c_str(), "rb"); From 3d2b884388ebf55081712f3be5794ec941bca9ff Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 6 Sep 2024 10:47:43 -0500 Subject: [PATCH 07/10] fix other test on centos7 --- src/nanoarrow/ipc/reader_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nanoarrow/ipc/reader_test.cc b/src/nanoarrow/ipc/reader_test.cc index 4af8d096a..929bde5f1 100644 --- a/src/nanoarrow/ipc/reader_test.cc +++ b/src/nanoarrow/ipc/reader_test.cc @@ -101,7 +101,7 @@ TEST(NanoarrowIpcReader, InputStreamBuffer) { TEST(NanoarrowIpcReader, InputStreamFile) { struct ArrowIpcInputStream stream; - ASSERT_EQ(ArrowIpcInputStreamInitFile(&stream, nullptr, 1), EINVAL); + ASSERT_NE(ArrowIpcInputStreamInitFile(&stream, nullptr, 1), NANOARROW_OK); uint8_t input_data[] = {0x01, 0x02, 0x03, 0x04, 0x05}; FILE* file_ptr = tmpfile(); From 82f67a68318527208c479dee6e507ac930958025 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 6 Sep 2024 11:10:14 -0500 Subject: [PATCH 08/10] maybe better approach for ernno-returning test --- src/nanoarrow/ipc/reader_test.cc | 3 ++- src/nanoarrow/ipc/writer_test.cc | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/nanoarrow/ipc/reader_test.cc b/src/nanoarrow/ipc/reader_test.cc index 929bde5f1..9dda66fdb 100644 --- a/src/nanoarrow/ipc/reader_test.cc +++ b/src/nanoarrow/ipc/reader_test.cc @@ -101,7 +101,8 @@ TEST(NanoarrowIpcReader, InputStreamBuffer) { TEST(NanoarrowIpcReader, InputStreamFile) { struct ArrowIpcInputStream stream; - ASSERT_NE(ArrowIpcInputStreamInitFile(&stream, nullptr, 1), NANOARROW_OK); + errno = EINVAL; + ASSERT_EQ(ArrowIpcInputStreamInitFile(&stream, nullptr, 1), EINVAL); uint8_t input_data[] = {0x01, 0x02, 0x03, 0x04, 0x05}; FILE* file_ptr = tmpfile(); diff --git a/src/nanoarrow/ipc/writer_test.cc b/src/nanoarrow/ipc/writer_test.cc index 5b06bc3b8..4cee56187 100644 --- a/src/nanoarrow/ipc/writer_test.cc +++ b/src/nanoarrow/ipc/writer_test.cc @@ -92,8 +92,9 @@ TEST(NanoarrowIpcWriter, OutputStreamFile) { TEST(NanoarrowIpcWriter, OutputStreamFileError) { nanoarrow::ipc::UniqueOutputStream stream; - EXPECT_NE(ArrowIpcOutputStreamInitFile(stream.get(), nullptr, /*close_on_release=*/1), - NANOARROW_OK); + errno = EINVAL; + EXPECT_EQ(ArrowIpcOutputStreamInitFile(stream.get(), nullptr, /*close_on_release=*/1), + EINVAL); auto phony_path = __FILE__ + std::string(".phony"); FILE* file_ptr = fopen(phony_path.c_str(), "rb"); From feae86bb9f6199dc4646ae835083c038f9e6b2e7 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 6 Sep 2024 11:16:08 -0500 Subject: [PATCH 09/10] update concurrency for long-running workflows --- .github/workflows/integration.yaml | 4 ++++ .github/workflows/verify.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 966ac4b4f..39be8c061 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -32,6 +32,10 @@ on: permissions: contents: read +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: integration: runs-on: ubuntu-latest diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index b234f535a..b754594e3 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -46,6 +46,10 @@ on: permissions: contents: read +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: verify: name: "${{ matrix.config.label }}" From 97cde5b1b04f7fda54f05fbbf7c94a6bdcb6e5df Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Wed, 11 Sep 2024 12:48:21 -0300 Subject: [PATCH 10/10] Update .github/workflows/integration.yaml Co-authored-by: Jacob Wujciak-Jens --- .github/workflows/integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 39be8c061..8a2e9d638 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -21,7 +21,7 @@ on: push: branches: - main - - 'maint-**' + - 'maint-*' pull_request: paths: - .github/workflows/integration.yaml