replace python ci script from shell to python

Signed-off-by: LiangliangSui <[email protected]>

.github/workflows/ci.yml

@@ -21,6 +21,7 @@ on:
   push:
     branches:
       - main
+      - feature
       - 'releases/**'
       - 'deploy/**'
       - 'test*'
@@ -149,22 +150,21 @@ jobs:
         run: python ./ci/run_ci.py cpp
   python:
     name: Python CI
-    # Fix python 3.6 install issue, see
-    # https://github.com/rwth-i6/returnn/commit/38ecab17d781c4b74db6a174c8097187380b4ddc
-    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.12]
+        os: [ubuntu-latest, macos-latest, windows-2022]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.12]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install bazel
-        run: ./ci/run_ci.sh install_bazel
       - name: Run Python CI
-        run: ./ci/run_ci.sh python
+        run: python ./ci/run_ci.py python
 
   go:
     name: Golang CI

ci/run_ci.py

@@ -23,12 +23,18 @@
 import os
 import logging
 import importlib
+import sys
 
 
 BAZEL_VERSION = "6.3.2"
 
 PYARROW_VERSION = "14.0.0"
 
+PYARROW_LEGACY_VERSION = "12.0.0"
+
+PROJECT_ROOT_DIR = os.path.join(os.path.split(os.path.realpath(__file__))[0], "../")
+
+
 logging.basicConfig(
     level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
 )
@@ -71,6 +77,10 @@ def _get_bazel_download_url():
     )
 
 
+def _cd_project_subdir(subdir):
+    os.chdir(os.path.join(PROJECT_ROOT_DIR, subdir))
+
+
 def _run_cpp():
     _install_cpp_deps()
     # run test
@@ -84,8 +94,7 @@ def _run_rust():
     _exec_cmd("rustup component add clippy-preview")
     _exec_cmd("rustup component add rustfmt")
     logging.info("Executing fury rust tests")
-    cur_script_abs_path = os.path.split(os.path.realpath(__file__))[0]
-    os.chdir(os.path.join(cur_script_abs_path, "../rust"))
+    _cd_project_subdir("rust")
 
     cmds = (
         "cargo doc --no-deps --document-private-items --all-features --open",
@@ -102,6 +111,15 @@ def _run_rust():
     logging.info("Executing fury rust tests succeeds")
 
 
+def _run_python():
+    logging.info("Executing fury python tests")
+    _install_bazel()
+    _install_pyfury()
+    _cd_project_subdir("python")
+    _exec_cmd("pytest -v -s --durations=60 pyfury/tests")
+    logging.info("Executing fury python tests succeeds")
+
+
 def _install_cpp_deps():
     _exec_cmd(f"pip install pyarrow=={PYARROW_VERSION}")
     _exec_cmd("pip install psutil")
@@ -133,6 +151,21 @@ def _install_bazel():
         file.write(f"\nbuild --jobs={limit_jobs}")
 
 
+def _install_pyfury():
+    _cd_project_subdir("python")
+    major = sys.version_info.major
+    minor = sys.version_info.minor
+    if major == 3:
+        pick_version = PYARROW_VERSION if minor >= 8 else PYARROW_LEGACY_VERSION
+    else:
+        pick_version = PYARROW_VERSION if major > 3 else PYARROW_LEGACY_VERSION
+    logging.info(f"pick pyarrow version: {pick_version}")
+
+    _exec_cmd(f"pip install pyarrow=={pick_version}")
+    _exec_cmd("pip install Cython wheel numpy pytest pandas setuptools")
+    _exec_cmd("pip install -v -e .")
+
+
 def _parse_args():
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter
@@ -156,6 +189,14 @@ def _parse_args():
     )
     rust_parser.set_defaults(func=_run_rust)
 
+    python_parser = subparsers.add_parser(
+        "python",
+        description="Run Python CI",
+        help="Run Python CI",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    python_parser.set_defaults(func=_run_python)
+
     args = parser.parse_args()
     arg_dict = dict(vars(args))
     del arg_dict["func"]