From 6aa02d6b3665cd48e14320880c77ad46ec532cfc Mon Sep 17 00:00:00 2001 From: Liangliang Sui Date: Fri, 5 Jan 2024 22:05:42 +0800 Subject: [PATCH] add ci Signed-off-by: Liangliang Sui --- .github/workflows/ci.yml | 308 +++++++++++++++--------------- bazel/arrow/pyarrow_configure.bzl | 28 ++- ci/run_ci.py | 119 ++++++++++++ src/fury/meta/preprocessor.h | 4 + src/fury/meta/type_traits.h | 3 + src/fury/row/row.cc | 4 +- 6 files changed, 304 insertions(+), 162 deletions(-) create mode 100644 ci/run_ci.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1da0c44a2c..7b69c89900 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ on: push: branches: - main + - dev - 'releases/**' - 'deploy/**' - 'test*' @@ -36,108 +37,108 @@ on: types: ['opened', 'reopened', 'synchronize'] jobs: - java: - name: Java CI - runs-on: ubuntu-latest - env: - MY_VAR: "PATH" - strategy: - matrix: - java-version: ["8", "11", "17", "21"] - steps: - - uses: actions/checkout@v3 - - name: Set up JDK ${{ matrix.java-version }} - uses: actions/setup-java@v3 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - - name: Install bazel - run: ./ci/run_ci.sh install_bazel - - name: Install python - run: ./ci/run_ci.sh install_python - - name: Install pyfury - run: ./ci/run_ci.sh install_pyfury - - name: Run CI with Maven - run: ./ci/run_ci.sh java${{ matrix.java-version }} + # java: + # name: Java CI + # runs-on: ubuntu-latest + # env: + # MY_VAR: "PATH" + # strategy: + # matrix: + # java-version: ["8", "11", "17", "21"] + # steps: + # - uses: actions/checkout@v3 + # - name: Set up JDK ${{ matrix.java-version }} + # uses: actions/setup-java@v3 + # with: + # java-version: ${{ matrix.java-version }} + # distribution: 'temurin' + # - name: Install bazel + # run: ./ci/run_ci.sh install_bazel + # - name: Install python + # run: ./ci/run_ci.sh install_python + # - name: Install pyfury + # run: ./ci/run_ci.sh install_pyfury + # - name: Run CI with Maven + # run: ./ci/run_ci.sh java${{ matrix.java-version }} - graalvm: - name: GraalVM CI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: graalvm/setup-graalvm@v1 - with: - java-version: '21' - distribution: 'graalvm' - github-token: ${{ secrets.GITHUB_TOKEN }} - native-image-job-reports: 'true' - - name: Build native image and run - run: ./ci/run_ci.sh graalvm_test + # graalvm: + # name: GraalVM CI + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - uses: graalvm/setup-graalvm@v1 + # with: + # java-version: '21' + # distribution: 'graalvm' + # github-token: ${{ secrets.GITHUB_TOKEN }} + # native-image-job-reports: 'true' + # - name: Build native image and run + # run: ./ci/run_ci.sh graalvm_test - scala: - name: Scala CI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up JDK8 - uses: actions/setup-java@v3 - with: - java-version: 8 - distribution: 'temurin' - - name: Install fury java - run: cd java && mvn -T10 clean install -DskipTests && cd - - - name: Test - run: | - # Avoid sbt download jackson-databind error - rm -rf /home/runner/.m2/repository/com/fasterxml/jackson/ - cd scala && sbt +test && cd - + # scala: + # name: Scala CI + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - name: Set up JDK8 + # uses: actions/setup-java@v3 + # with: + # java-version: 8 + # distribution: 'temurin' + # - name: Install fury java + # run: cd java && mvn -T10 clean install -DskipTests && cd - + # - name: Test + # run: | + # # Avoid sbt download jackson-databind error + # rm -rf /home/runner/.m2/repository/com/fasterxml/jackson/ + # cd scala && sbt +test && cd - - integration_tests: - name: integration_tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up JDK8 - uses: actions/setup-java@v3 - with: - java-version: 8 - distribution: 'temurin' - - name: Install fury-benchmark jar - run: | - cd java && mvn -T10 -B --no-transfer-progress clean install -DskipTests && cd - - cd java/fury-benchmark && mvn -T10 -B --no-transfer-progress clean install -DskipTests -Pjmh && cd - - - name: Run CI - run: ./ci/run_ci.sh integration_tests + # integration_tests: + # name: integration_tests + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - name: Set up JDK8 + # uses: actions/setup-java@v3 + # with: + # java-version: 8 + # distribution: 'temurin' + # - name: Install fury-benchmark jar + # run: | + # cd java && mvn -T10 -B --no-transfer-progress clean install -DskipTests && cd - + # cd java/fury-benchmark && mvn -T10 -B --no-transfer-progress clean install -DskipTests -Pjmh && cd - + # - name: Run CI + # run: ./ci/run_ci.sh integration_tests - javascript: - name: Javascript CI - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [14.x, 20.x] - steps: - - uses: actions/checkout@v3 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - name: Run CI with NodeJS - run: ./ci/run_ci.sh javascript + # javascript: + # name: Javascript CI + # runs-on: ubuntu-latest + # strategy: + # matrix: + # node-version: [14.x, 20.x] + # steps: + # - uses: actions/checkout@v3 + # - name: Use Node.js ${{ matrix.node-version }} + # uses: actions/setup-node@v3 + # with: + # node-version: ${{ matrix.node-version }} + # - name: Run CI with NodeJS + # run: ./ci/run_ci.sh javascript - rust: - name: Rust CI - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@nightly - - run: ./ci/run_ci.sh rust + # rust: + # name: Rust CI + # runs-on: ubuntu-latest + # timeout-minutes: 45 + # steps: + # - uses: actions/checkout@v3 + # - uses: dtolnay/rust-toolchain@nightly + # - run: ./ci/run_ci.sh rust cpp: name: C++ CI strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-2022] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -146,67 +147,70 @@ jobs: with: python-version: 3.8 - name: Install bazel - run: ./ci/run_ci.sh install_bazel + run: | + pip install psutil + pip install pyarrow==14.0.0 + python ./ci/run_ci.py install-bazel - name: Run C++ CI with Bazel - run: ./ci/run_ci.sh cpp + run: python ./ci/run_ci.py cpp - python: - name: Python CI - # Fix python 3.6 install issue, see - # https://github.com/rwth-i6/returnn/commit/38ecab17d781c4b74db6a174c8097187380b4ddc - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.7, 3.12] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install bazel - run: ./ci/run_ci.sh install_bazel - - name: Run Python CI - run: ./ci/run_ci.sh python + # python: + # name: Python CI + # # Fix python 3.6 install issue, see + # # https://github.com/rwth-i6/returnn/commit/38ecab17d781c4b74db6a174c8097187380b4ddc + # runs-on: ubuntu-20.04 + # strategy: + # matrix: + # python-version: [3.7, 3.12] + # steps: + # - uses: actions/checkout@v3 + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v2 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install bazel + # run: ./ci/run_ci.sh install_bazel + # - name: Run Python CI + # run: ./ci/run_ci.sh python - go: - name: Golang CI - runs-on: ubuntu-latest - strategy: - matrix: - go-version: [ '1.13', '1.18' ] - steps: - - uses: actions/checkout@v4 - - name: Setup Go ${{ matrix.go-version }} - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - - name: Display Go version - run: go version - - name: Install bazel - run: ./ci/run_ci.sh install_bazel - - name: Install python - run: ./ci/run_ci.sh install_python - - name: Install pyfury - run: ./ci/run_ci.sh install_pyfury - - name: Run Golang CI - run: ./ci/run_ci.sh go + # go: + # name: Golang CI + # runs-on: ubuntu-latest + # strategy: + # matrix: + # go-version: [ '1.13', '1.18' ] + # steps: + # - uses: actions/checkout@v4 + # - name: Setup Go ${{ matrix.go-version }} + # uses: actions/setup-go@v4 + # with: + # go-version: ${{ matrix.go-version }} + # - name: Display Go version + # run: go version + # - name: Install bazel + # run: ./ci/run_ci.sh install_bazel + # - name: Install python + # run: ./ci/run_ci.sh install_python + # - name: Install pyfury + # run: ./ci/run_ci.sh install_pyfury + # - name: Run Golang CI + # run: ./ci/run_ci.sh go - lint: - name: Code Style Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up JDK ${{ matrix.java-version }} - uses: actions/setup-java@v3 - with: - java-version: 21 - distribution: 'oracle' - - name: Check License Header - uses: korandoru/hawkeye@v3 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Check code style - run: ./ci/run_ci.sh format + # lint: + # name: Code Style Check + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - name: Set up JDK ${{ matrix.java-version }} + # uses: actions/setup-java@v3 + # with: + # java-version: 21 + # distribution: 'oracle' + # - name: Check License Header + # uses: korandoru/hawkeye@v3 + # - name: Set up Python 3.8 + # uses: actions/setup-python@v2 + # with: + # python-version: 3.8 + # - name: Check code style + # run: ./ci/run_ci.sh format diff --git a/bazel/arrow/pyarrow_configure.bzl b/bazel/arrow/pyarrow_configure.bzl index c915097e08..ca4fd2f20b 100644 --- a/bazel/arrow/pyarrow_configure.bzl +++ b/bazel/arrow/pyarrow_configure.bzl @@ -145,6 +145,15 @@ def _symlink_genrule_for_dir( src_files = files.splitlines() command = [] outs = [] + + # if _is_windows(repository_ctx): + # # print("is windows") + # dest_files = [ file.replace("/", "\\") for file in dest_files] + # # print(dest_files) + # src_files = [ file.replace("/", "\\") for file in src_files] + + # print(dest_files) + for i in range(len(dest_files)): if dest_files[i] != "": # If we have only one file to link we do not want to use the dest_dir, as @@ -153,8 +162,8 @@ def _symlink_genrule_for_dir( # Copy the headers to create a sandboxable setup. cmd = "cp -f" - command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) - outs.append(' "' + dest_dir + dest_files[i] + '",') + command.append(cmd + ' "%s" "%s"' % (src_files[i].replace("\\", "\\\\"), dest.replace("\\", "\\\\"))) + outs.append(' "' + dest_dir + dest_files[i].replace("\\", "\\\\") + '",') genrule = _genrule( genrule_name, " && ".join(command), @@ -162,7 +171,7 @@ def _symlink_genrule_for_dir( ) return genrule -def _get_pyarrow_include(repository_ctx, python_bin="python3"): +def _get_pyarrow_include(repository_ctx, python_bin="python"): """Gets the pyarrow include path.""" result = _execute( repository_ctx, [ @@ -174,10 +183,10 @@ def _get_pyarrow_include(repository_ctx, python_bin="python3"): + python_bin + ".) " + "Is distutils installed?")) return result.stdout.splitlines()[0] -def _get_pyarrow_shared_library(repository_ctx, library_name, python_bin="python3"): +def _get_pyarrow_shared_library(repository_ctx, library_name, python_bin="python"): """Gets the pyarrow shared library path.""" code = """import pyarrow, os, glob;print(glob.glob(os.path.join(""" +\ - """os.path.dirname(pyarrow.__file__), 'lib{}.*'))[0])""".format(library_name) + """os.path.dirname(pyarrow.__file__), '*{}.*'))[0])""".format(library_name) result = _execute( repository_ctx, [ python_bin, "-c", code @@ -189,7 +198,7 @@ def _get_pyarrow_shared_library(repository_ctx, library_name, python_bin="python return result.stdout.splitlines()[0] #python numpy include -def _get_python_numpy_include(repository_ctx, python_bin="python3"): +def _get_python_numpy_include(repository_ctx, python_bin="python"): """Gets the python numpy include path.""" result = _execute( repository_ctx, [ @@ -210,13 +219,16 @@ def _pyarrow_pip_impl(repository_ctx): "arrow_header_include", ) + split_letter = "\\" if _is_windows(repository_ctx) else "/" + # split_letter = "/" + arrow_library_path = _get_pyarrow_shared_library(repository_ctx, "arrow") - arrow_library = arrow_library_path.rsplit("/",1 )[-1] + arrow_library = arrow_library_path.rsplit(split_letter,1 )[-1] arrow_library_rule = _symlink_genrule_for_dir( repository_ctx, None, "", "libarrow", [arrow_library_path], [arrow_library]) arrow_python_library_path = _get_pyarrow_shared_library(repository_ctx, "arrow_python") - arrow_python_library = arrow_python_library_path.rsplit("/",1 )[-1] + arrow_python_library = arrow_python_library_path.rsplit(split_letter,1 )[-1] arrow_python_library_rule = _symlink_genrule_for_dir( repository_ctx, None, "", "libarrow_python", [arrow_python_library_path], [arrow_python_library]) diff --git a/ci/run_ci.py b/ci/run_ci.py new file mode 100644 index 0000000000..5f06cae0b9 --- /dev/null +++ b/ci/run_ci.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import argparse +import subprocess +import platform +import psutil +import urllib.request as ulib +import os +import logging +import sys + +BAZEL_VERSION = "6.3.2" + + +def _exec_cmd(cmd: str): + logging.info(f"running command: {cmd}") + print(f"running command: {cmd}") + try: + result = subprocess.check_output(cmd, shell=True).decode("GBK") + except subprocess.CalledProcessError as e: + print(f"sll-error: {e.output.decode()}") + raise + logging.info(f"command result: {result}") + print(f"command result: {result}") + return result + + +def _get_os_name_lower(): + return platform.system().lower() + + +def _get_os_machine(): + machine = platform.machine().lower() + # windows return amd64, so replace it to x86_64 + return machine.replace("amd64", "x86_64") + + +def _install_bazel(): + os_name = _get_os_name_lower() + is_windows = os_name == "windows" + download_url_base = f"https://github.com/bazelbuild/bazel/releases/download/{BAZEL_VERSION}" + local_name = "bazel.exe" if is_windows else "bazel-installer.sh" + exe_tail = "exe" if is_windows else "sh" + bazel_download_url = f"{download_url_base}/bazel-{BAZEL_VERSION}{'' if is_windows else '-installer'}-" \ + f"{os_name}-{_get_os_machine()}.{exe_tail}" + print(bazel_download_url) + ulib.urlretrieve(bazel_download_url, local_name) + + if is_windows: + bazel_path = os.path.join(os.getcwd(), local_name) + _exec_cmd(f"setx path \"%PATH%;{bazel_path}\"") + else: + os.chmod(local_name, 0o777) + _exec_cmd(f"./{local_name} --user") + os.remove(local_name) + + # bazel install status check + _exec_cmd("bazel --version") + + # default is byte + total_mem = psutil.virtual_memory().total + limit_jobs = int(total_mem / 1024 / 1024 / 1024 / 3) + with open(".bazelrc", "a") as file: + file.write(f"\nbuild --jobs={limit_jobs}") + + +def _run_cpp(): + # run test + query_result = _exec_cmd("bazel query //...") + _exec_cmd("bazel test {}".format(query_result.replace("\n", " ").replace("\r", " "))) + + +def _parse_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.set_defaults(func=parser.print_help) + subparsers = parser.add_subparsers() + + cpp_parser = subparsers.add_parser( + "cpp", + description="Run C++ CI", + help="Run C++ CI", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + cpp_parser.set_defaults(func=_run_cpp) + + bazel_parser = subparsers.add_parser( + "install-bazel", + description="Install bazel on the current machine", + help="Install bazel on the current machine", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + bazel_parser.set_defaults(func=_install_bazel) + + args = parser.parse_args() + arg_dict = dict(vars(args)) + del arg_dict["func"] + args.func(**arg_dict) + + +if __name__ == "__main__": + _parse_args() diff --git a/src/fury/meta/preprocessor.h b/src/fury/meta/preprocessor.h index 97ba6e19a7..a869c85437 100644 --- a/src/fury/meta/preprocessor.h +++ b/src/fury/meta/preprocessor.h @@ -18,6 +18,8 @@ */ #pragma once +#ifndef PREPROCESSOR_HEAD +#define PREPROCESSOR_HEAD #define FURY_PP_CONCAT_IMPL(a, b) a##b @@ -202,3 +204,5 @@ #define FURY_PP_FOREACH_1_IMPL_62(X, C, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62) X(C, _1) X(C, _2) X(C, _3) X(C, _4) X(C, _5) X(C, _6) X(C, _7) X(C, _8) X(C, _9) X(C, _10) X(C, _11) X(C, _12) X(C, _13) X(C, _14) X(C, _15) X(C, _16) X(C, _17) X(C, _18) X(C, _19) X(C, _20) X(C, _21) X(C, _22) X(C, _23) X(C, _24) X(C, _25) X(C, _26) X(C, _27) X(C, _28) X(C, _29) X(C, _30) X(C, _31) X(C, _32) X(C, _33) X(C, _34) X(C, _35) X(C, _36) X(C, _37) X(C, _38) X(C, _39) X(C, _40) X(C, _41) X(C, _42) X(C, _43) X(C, _44) X(C, _45) X(C, _46) X(C, _47) X(C, _48) X(C, _49) X(C, _50) X(C, _51) X(C, _52) X(C, _53) X(C, _54) X(C, _55) X(C, _56) X(C, _57) X(C, _58) X(C, _59) X(C, _60) X(C, _61) X(C, _62) #define FURY_PP_FOREACH_1_IMPL_63(X, C, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63) X(C, _1) X(C, _2) X(C, _3) X(C, _4) X(C, _5) X(C, _6) X(C, _7) X(C, _8) X(C, _9) X(C, _10) X(C, _11) X(C, _12) X(C, _13) X(C, _14) X(C, _15) X(C, _16) X(C, _17) X(C, _18) X(C, _19) X(C, _20) X(C, _21) X(C, _22) X(C, _23) X(C, _24) X(C, _25) X(C, _26) X(C, _27) X(C, _28) X(C, _29) X(C, _30) X(C, _31) X(C, _32) X(C, _33) X(C, _34) X(C, _35) X(C, _36) X(C, _37) X(C, _38) X(C, _39) X(C, _40) X(C, _41) X(C, _42) X(C, _43) X(C, _44) X(C, _45) X(C, _46) X(C, _47) X(C, _48) X(C, _49) X(C, _50) X(C, _51) X(C, _52) X(C, _53) X(C, _54) X(C, _55) X(C, _56) X(C, _57) X(C, _58) X(C, _59) X(C, _60) X(C, _61) X(C, _62) X(C, _63) // clang-format on + +#endif \ No newline at end of file diff --git a/src/fury/meta/type_traits.h b/src/fury/meta/type_traits.h index e5c02feaf5..8614b6fef4 100644 --- a/src/fury/meta/type_traits.h +++ b/src/fury/meta/type_traits.h @@ -18,6 +18,8 @@ */ #pragma once +#ifndef SLL_TEST +#define SLL_TEST #include #include @@ -123,3 +125,4 @@ constexpr inline bool IsPairIterable = } // namespace meta } // namespace fury +#endif diff --git a/src/fury/row/row.cc b/src/fury/row/row.cc index 79fa6594d3..85c50d2c5d 100644 --- a/src/fury/row/row.cc +++ b/src/fury/row/row.cc @@ -237,8 +237,8 @@ int *ArrayData::GetDimensions(ArrayData &array, int num_dims) { // use deep-first search to search to numDimensions-1 layer to get dimensions. int depth = 0; auto dimensions = new int[num_dims]; - int start_from_lefts[num_dims]; - ArrayData *arrs[num_dims]; // root to current node + int *start_from_lefts = new int[num_dims]; + ArrayData **arrs = new ArrayData*[num_dims]; // root to current node ArrayData &arr = array; while (depth < num_dims) { arrs[depth] = &arr;