Merge pull request #262 from ecmwf-ifs/nabr-macos-ci

Run pytest on macos in GH actions

.github/workflows/tests.yml

@@ -15,13 +15,36 @@ on:
 
 jobs:
   pytest:
-    name: Python ${{ matrix.python-version }}
+    name: pytest
 
-    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false  # false: try to complete all jobs
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        name:
+          - linux python-3.8
+          - linux python-3.9
+          - linux python-3.10
+          - linux python-3.11
+          - macos python-3.11
+
+        include:
+          - name: linux python-3.8
+            os: ubuntu-20.04
+            python-version: '3.8'
+          - name: linux python-3.9
+            os: ubuntu-20.04
+            python-version: '3.9'
+          - name: linux python-3.10
+            os: ubuntu-20.04
+            python-version: '3.10'
+          - name: linux python-3.11
+            os: ubuntu-20.04
+            python-version: '3.11'
+          - name: macos python-3.11
+            os: macos-14
+            python-version: '3.11'
+
+    runs-on: ${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v4
@@ -34,21 +57,35 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
 
-      - name: Refresh package lists
+      - name: Refresh package lists and install Loki dependencies
         run: |
-          sudo apt-get -o Acquire::Retries=3 update || true
-
-      - name: Install Loki dependencies
-        run: |
-          sudo apt-get -o Acquire::Retries=3 install -y graphviz gfortran
-
-      - name: Install OMNI + CLAW dependencies
+          if [[ "${{ matrix.os }}" =~ macos ]]; then
+            brew install gcc@13 graphviz
+          else
+            sudo apt-get -o Acquire::Retries=3 update || true
+            sudo apt-get -o Acquire::Retries=3 install -y graphviz gfortran
+          fi
+
+      - name: Install OMNI + CLAW dependencies on Linux
+        if: ${{ ! startsWith(matrix.os, 'macos') }}
         run: |
           sudo apt-get -o Acquire::Retries=3 install -y byacc flex openjdk-11-jdk cmake ant
 
       - name: Install Loki
         run: |
-          ./install --with-claw --with-ofp --with-examples --with-tests --with-dace
+          if [[ "${{ matrix.os }}" =~ macos ]]; then
+            export CC=gcc-13 CXX=g++-13 FC=gfortran-13
+            ./install --with-examples --with-tests --with-dace
+            echo "export PATH=$(brew --prefix)/opt/[email protected]/libexec/bin:$(brew --prefix)/bin:${PATH}" | cat - loki-activate > loki-activate.tmp
+            mv loki-activate.tmp loki-activate
+            echo "export CC=gcc-13" >> loki-activate
+            echo "export CXX=g++-13" >> loki-activate
+            echo "export FC=gfortran-13" >> loki-activate
+            echo "export F90=gfortran-13" >> loki-activate
+            echo "export LD=gfortran-13" >> loki-activate
+          else
+            ./install --with-claw --with-ofp --with-examples --with-tests --with-dace
+          fi
 
       - name: Install up-to-date CMake
         run: |
@@ -62,6 +99,7 @@ jobs:
 
       - name: Upload loki coverage report to Codecov
         uses: codecov/codecov-action@v4
+        if: ${{ ! startsWith(matrix.os, 'macos') }}
         with:
           flags: loki
           files: ./coverage.xml
@@ -75,6 +113,7 @@ jobs:
 
       - name: Upload lint_rules coverage report to Codecov
         uses: codecov/codecov-action@v4
+        if: ${{ ! startsWith(matrix.os, 'macos') }}
         with:
           flags: lint_rules
           files: ./coverage.xml

CMakeLists.txt

@@ -34,8 +34,9 @@
 # Version 3.12 required to use FindPython
 # Version 3.15 officially required to use Python3_FIND_VIRTUALENV (not working on 3.15.3,
 # though, and use 3.17 for conda support anyway)
-cmake_minimum_required( VERSION 3.17 FATAL_ERROR )
-find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild )
+# Version 3.19 for support of find_package version range and file(CHMOD)
+cmake_minimum_required( VERSION 3.19 FATAL_ERROR )
+find_package( ecbuild 3.7 REQUIRED )
 
 # Specify project and configuration options
 project( loki LANGUAGES NONE )
@@ -100,7 +101,8 @@ if( NOT HAVE_NO_INSTALL )
     # Setup Python virtual environment
     include( python_venv )
     set( loki_VENV_PATH ${CMAKE_CURRENT_BINARY_DIR}/loki_env )
-    setup_python_venv( ${loki_VENV_PATH} )
+    set( PYTHON_VERSION 3.8...<3.12 )
+    setup_python_venv( ${loki_VENV_PATH} ${PYTHON_VERSION} )
     install( DIRECTORY ${loki_VENV_PATH} DESTINATION . USE_SOURCE_PERMISSIONS )
 
     # Enable Pytest tests as ecbuild/ctest targets
@@ -171,13 +173,13 @@ if( NOT HAVE_NO_INSTALL )
     # We install Loki at configure time (for now), since bulk-transformation planning
     # requires configure time execution to allow injection with CMake targets.
 
-    message( STATUS "Install Loki in virtual environment" )
+    ecbuild_info( "Install Loki in virtual environment" )
     execute_process( COMMAND ${Python3_EXECUTABLE} -m pip install ${PIP_OPTIONS} ${CMAKE_CURRENT_SOURCE_DIR}${_INSTALL_OPTIONS} )
-    message( STATUS "Install Loki in virtual environment - done" )
+    ecbuild_info( "Install Loki in virtual environment - done" )
 
-    message( STATUS "Install Loki lint_rules in virtual environment" )
+    ecbuild_info( "Install Loki lint_rules in virtual environment" )
     execute_process( COMMAND ${Python3_EXECUTABLE} -m pip install ${PIP_OPTIONS} ${CMAKE_CURRENT_SOURCE_DIR}/lint_rules )
-    message( STATUS "Install Loki lint_rules in virtual environment - done" )
+    ecbuild_info( "Install Loki lint_rules in virtual environment - done" )
 
     # Make the shebang in Python scripts use relative paths
     list(

INSTALL.md

@@ -19,14 +19,19 @@ do this automatically.
 ## Requirements
 
 - Python 3.8+ with virtualenv and pip
-- For OpenFortranParser/OMNI+CLAW:
-  - JDK 1.8+ with ant (can be installed using the install script or ecbuild)
-  - libxml2 (with headers)
-- For graphical output from the scheduler: graphviz
+- For graphical output of Scheduler dependency graphs: graphviz
 
-## Installation via `pip install`
+### Optional requirements
 
-The easiest way to obtain a useable installation of Loki with the fparser frontend:
+The following is required to use the Open Fortran Parser (OFP) or OMNI frontend or the CLAW compiler:
+
+- JDK 1.8+ with ant (can be installed using the install script or ecbuild)
+- libxml2 (with headers)
+
+## Installation without prior download
+
+The easiest way to obtain a useable installation of Loki with the fparser frontend does not
+require downloading the source code. Simply run the following commands:
 
 ```bash
 python3 -m venv loki_env  # Create a virtual environment
@@ -35,17 +40,50 @@ source loki_env/bin/activate  # Activate the virtual environment
 # Installation of the Loki core library
 pip install "loki @ git+https://github.com/ecmwf-ifs/loki.git"
 
-# Optional: Installation of additional transformations
-pip install "transformations @ git+https://github.com/ecmwf-ifs/loki.git#subdirectory=transformations"
-
-# Optional: Installation of rules for the use as a linter
+# Optional: Installation of IFS lint rules for the use as a linter
 pip install "lint_rules @ git+https://github.com/ecmwf-ifs/loki.git#subdirectory=lint_rules"
 ```
 
 This makes the Python package available and installs the scripts `loki-transform.py` and `loki-lint.py` on the PATH.
 
 
-## Installation using install script
+## Installation from source
+
+After downloading the source code, e.g., via
+
+```bash
+git clone https://github.com/ecmwf-ifs/loki.git
+```
+
+enter the created source directory and choose one of the following installation methods.
+
+### Installation with pip
+
+This yields an installation that uses the fparser frontend and is suitable for
+development of transformations and working with the example notebooks:
+
+```bash
+python3 -m venv loki_env  # Create a virtual environment
+source loki_env/bin/activate  # Activate the virtual environment
+
+# Installation of the Loki core library
+# Optional:
+#   * Add `-e` to obtain an editable install that allows modifying the
+#     source files without having to re-install the package
+#   * Enable the following options by providing them as a comma-separated
+#     list in square brackets behind the `.`:
+#     * tests    - allows running the Loki test suite
+#     * examples - installs dependencies to run the example notebooks
+#     * docs     - installs dependencies required to generate the Sphinx documentation
+#     * dace     - installs DaCe
+pip install .
+
+# Optional: Installation of IFS lint rules for the use as a linter
+#           (again optionally with `-e` for an editable install)
+pip install ./lint_rules
+```
+
+### Installation using install script
 
 The provided `install` script can be used to install Loki with selected
 dependencies inside a local virtual environment `loki_env`. This is the
@@ -85,7 +123,7 @@ will install only the Fparser2 frontend.
 After completion, this script writes a `loki-activate` file that can be sourced
 to bring up the virtual environment and set paths for the external dependencies.
 
-### Examples:
+#### Examples:
 
 The default command on ECMWF's Atos HPC facility for a full stack installation is
 
@@ -105,7 +143,7 @@ To update the installation (e.g., to add JDK), the existing virtual environment
 ./install --with-claw --with-jdk --with-ant --use-venv=loki_env --with-ofp
 ```
 
-## Installation using CMake/ecbuild
+### Installation using CMake/ecbuild
 
 Loki and dependencies (excluding OpenFortranParser) can be installed using
 [ecbuild](https://github.com/ecmwf/ecbuild) (a set of CMake macros and a wrapper
@@ -167,6 +205,40 @@ need to take care of PATH handling on the user side. See the [CLOUDSC
 dwarf](https://github.com/ecmwf-ifs/dwarf-p-cloudsc) for an example how this can
 be used.
 
+## Installation on MacOS
+
+Although tailored to the Linux environment commonly found on HPC systems, Loki
+can also be installed on MacOS.
+
+This requires installing some additional dependencies using
+[Brew](https://brew.sh) to allow running the Loki test suite:
+
+
+```bash
+# Install dependencies with brew
+brew install gcc@13 graphviz [email protected]
+
+# Install Loki using the install script
+# NB: we explicitly select Python 3.11 (in case a newer version is the default)
+#     by adding it in first place to the search path
+PATH="$(brew --prefix)/opt/[email protected]/libexec/bin:$PATH" \
+  CC=gcc-13 CXX=g++-13 FC=gfortran-13 \
+  ./install --with-examples --with-tests --with-dace
+
+# Amend the Loki environment with correct compiler variables
+echo "export PATH=$(brew --prefix)/opt/[email protected]/libexec/bin:$(brew --prefix)/bin:${PATH}" | cat - loki-activate > loki-activate.tmp
+mv loki-activate.tmp loki-activate
+echo "export CC=gcc-13" >> loki-activate
+echo "export CXX=g++-13" >> loki-activate
+echo "export FC=gfortran-13" >> loki-activate
+echo "export F90=gfortran-13" >> loki-activate
+echo "export LD=gfortran-13" >> loki-activate
+
+# Activate the virtual environment to run the tests
+source loki-activate
+pytest --pyargs loki
+```
+
 ## Installation as part of an ecbundle bundle
 
 Loki being installable by CMake/ecbuild makes it easy to integrate with
@@ -217,7 +289,6 @@ that has support for editable installs using `pyproject.toml`.
 ```bash
 pushd loki
 pip install -e .[tests,examples]
-pip install -e ./transformations
 pip install -e ./lint_rules
 popd
 ```