[newchem-cpp] Introduce Installation Tests

.circleci/config.yml

@@ -1,11 +1,11 @@
 version: 2.1
 
 commands:
-  set-env:
-    description: "Set environment variables."
+  setup-system-for-pytest:
+    description: "Set up the system for pytest."
     steps:
       - run:
-          name: "Set environment variables."
+          name: "Setup System"
           command: |
             echo "backend : Agg" > $HOME/matplotlibrc
             echo 'export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH' >> $BASH_ENV
@@ -17,27 +17,10 @@ commands:
               git fetch --tags https://github.com/$CIRCLE_PR_USERNAME/$CIRCLE_PR_REPONAME
             fi
 
-  install-cmake-dependencies:
-    description: "Install minimal dependencies needed for a CMake build"
-    steps:
-      - run:
-          name: "Install minimal dependencies needed for a CMake build"
-          command: |
             git submodule update --init --remote
-            sudo apt update
-            sudo apt install -y libhdf5-serial-dev gfortran ninja-build cmake
+            ./scripts/ci/install_dependencies.py --color \
+              --preset classic-build --preset cmake-build
 
-  install-dependencies:
-    description: "Install dependencies."
-    steps:
-      - install-cmake-dependencies
-      - run:
-          name: "Install all other dependencies."
-          command: |
-            source $BASH_ENV
-            # libtool-bin is a requirement of the classic build-system
-            # castxml is a requirement of the gracklepy test-suite
-            sudo apt install -y libtool-bin castxml
             python3 -m venv $HOME/venv
             source $HOME/venv/bin/activate
             pip install --upgrade pip
@@ -270,8 +253,7 @@ jobs:
 
     steps:
       - checkout
-      - set-env
-      - install-dependencies
+      - setup-system-for-pytest
       - download-test-data
       - store-gracklepy-suite-gold-standard-answers
 
@@ -316,20 +298,20 @@ jobs:
 
     steps:
       - checkout
-      - install-cmake-dependencies
-      - run:
-          name: "update CMake to newer version"
-          command: |
-            sudo apt remove cmake -y
-            sudo apt install python3-pip
-            # this is a hack to install a newer version of CMake (through PyPI)
-            pip3 install cmake
+      # we pinned cmake to an arbitrarily new version
+      - run: |
+          git submodule update --init --remote
+          ./scripts/ci/install_dependencies.py --color \
+              --preset corelib-tests --preset static-analysis \
+              --software cmake=4.2 --software clang-tidy=20 \
+              --manual-install-dir ~/local_deps
 
       - run:
           name: "Build libgrackle and all of the core library's tests"
           command: |
             cmake --preset ci-linux
             cmake --build build_ci-linux
+
       - run:
           name: "Run the suite of tests that operate directly on the core-library."
           command: ctest --test-dir build_ci-linux --output-on-failure
@@ -348,72 +330,6 @@ jobs:
       #    environment:
       #      OMP_NUM_THREADS: 4
 
-
-      # the following step currently uses the llvm hosted repositories
-      # (described at https://apt.llvm.org/)
-      # -> if the day ever comes where we need to install using a different
-      #    mechanism, we could always install it from the package on PyPI
-      # -> we could also leverage the current installation approach to make
-      #    test-builds with llvm
-      - run:
-          name: "Install clang-tidy"
-          command: |
-            # at the moment, we are trying to keep this synchronized with the
-            # version of clang-format used in pre-commit
-            LLVM_VERSION=20
-
-            # todo: maybe move this logic into a script
-
-            # in general we could replace a lot of the following with with the
-            # script downloaded via `wget https://apt.llvm.org/llvm.sh`
-            # -> it seems like the script may do a lot of extra unnecessary work
-
-            # Step 1: register LLVM's official apt registry
-            # =============================================
-
-            # Step 1a: add the llvm key for authenticating packages
-            GPG_DST=/etc/apt/trusted.gpg.d/apt.llvm.org.asc
-            GPG_SRC_URL="https://apt.llvm.org/llvm-snapshot.gpg.key"
-            wget -qO- "${GPG_SRC_URL}" | sudo tee "${GPG_DST}"
-
-            # Step 1b: lookup the codename of the distribution (e.g. jammy)
-            # -> Debian: only provides VERSION_CODENAME
-            # -> vanilla Ubuntu: UBUNTU_CODENAME & VERSION_CODENAME are same
-            # -> distinction is more important for (some) Ubuntu derivatives
-            source <(grep -F 'UBUNTU_CODENAME' /etc/os-release)
-            CODENAME="${UBUNTU_CODENAME}"
-
-            # step 1c: record the actual source of the package
-            # -> the formula for determining the uri and suite-name is fairly
-            #    consistent unless you are describing Debian Testing or the
-            #    LLVM version under active development
-            uri="http://apt.llvm.org/${CODENAME}/"
-            suite="llvm-toolchain-${CODENAME}-${LLVM_VERSION}"
-            sudo add-apt-repository \
-                --yes \
-                --sourceslist "deb ${uri} ${suite} main"
-
-            # Step 2: actually download clang-tidy
-            # ====================================
-            sudo apt-get update
-            sudo apt-get install -y clang-tidy-${LLVM_VERSION}
-
-            # Step 3: Santity Check and make a symlink
-            # ========================================
-            # The symlink is VERY hacky, but it lets us avoid carrying the
-            # LLVM_VERSION variable between job steps
-            bin_path="/usr/bin/clang-tidy-${LLVM_VERSION}"
-            if [ ! -f ${bin_path} ]; then
-              echo "Our assumptions about where the file is installed is wrong"
-              exit 1
-            elif [ -f /usr/bin/clang-tidy ]; then
-              echo "another version of clang-tidy seems to be installed"
-              exit 1
-            else
-              # this is pretty hacky!
-              sudo ln -s ${bin_path} /usr/bin/clang-tidy
-            fi
-
       - run:
           name: "Run clang-tidy"
           command: |
@@ -425,6 +341,27 @@ jobs:
             cmake --build build_ci-linux-nowarn
 
 
+  installtest:
+
+    executor:
+      name: python
+      tag: 3.7.17
+      resource_class: small  # <- 1 core
+
+    working_directory: ~/grackle
+
+    steps:
+      - checkout
+      - run: git submodule update --init --remote
+
+      # I believe that the following command makes docker available in the
+      # environment where we are executing commands
+      # -> this is only needed since the job's executor is a Docker container
+      - setup_remote_docker:
+          docker_layer_caching: true
+
+      - run: tests/install-tests/installtest.py exec
+
   docs-build:
     parameters:
       tag:
@@ -442,6 +379,7 @@ jobs:
       - install-docs-dependencies
       - build-docs
 
+
 workflows:
    version: 2
 
@@ -454,6 +392,9 @@ workflows:
        - corelib-tests:
            name: "Core library test suite"
 
+       - installtest:
+           name: "Test linking against various configurations of core library"
+
        - docs-build:
            name: "Docs build"
            tag: "3.10.3"
@@ -474,6 +415,9 @@ workflows:
        - corelib-tests:
            name: "Core library test suite"
 
+       - installtest:
+           name: "Installtest"
+
        - docs-build:
            name: "Docs build"
            tag: "3.10.3"

.gitignore

@@ -49,6 +49,7 @@ src/python/.cache/*
 src/python/pygrackle.egg-info/*
 src/python/pygrackle/*.c
 src/python/tests/test_answers
+.pytest_cache/
 
 # build-directories for documentation
 doc/build
@@ -67,6 +68,10 @@ uv.lock
 
 # IDE stuff
 .idea/
+.vscode/
+
+# created by the clangd language server
+.cache/
 
 # Environments
 .env

cmake/GrackleConfig.cmake.in

@@ -251,30 +251,46 @@ if (${_GRACKLE_LIBKIND} STREQUAL "static")
 
   unset(_FIND_H5_ARGS)
 
-  if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY)
-    list(APPEND _EXTRA_ARGS QUIET)
-  endif()
-
-  if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED)
-    list(APPEND _EXTRA_ARGS REQUIRED)
-  endif()
+  # the order in which we build up _FIND_H5_ARGS matters!
 
   # if we know the HDF5 version used to build to build Grackle as a static
-  # library, we really need to make sure the version number of the hdf5 library
-  # match the version number of the hdf5-headers (the documentation for the
-  # H5check_version function makes it clear a mismatch is bad!)
+  # library, we really should try to request it
   set(_DESIRED_HDF5_VERSION "@HDF5_VERSION@")
   if(NOT "${_DESIRED_HDF5_VERSION}" STREQUAL "")
-    list(APPEND _EXTRA_ARGS "${_DESIRED_HDF5_VERSION}" EXACT)
+    list(APPEND _FIND_H5_ARGS "${_DESIRED_HDF5_VERSION}")
+    if ("${_DESIRED_HDF5_VERSION}" VERSION_LESS "2.0.0")
+      # prior to version 2.0, HDF5 did not adhere to semver
+      # -> in this case, we REALLY need to make sure the version number of the
+      #    hdf5 library matches the version number of the hdf5-headers (the
+      #    documentation for the H5check_version function makes it clear a
+      #    mismatch is bad!)
+      list(APPEND _FIND_H5_ARGS EXACT)
+    else()
+      # do NOT append EXACT
+      # -> starting with HDF5 2.0, HDF5 is only ever built with CMake and we
+      #    can rely upon the version compatability checks provided in HDF5's
+      #    Config Package File
+    endif()
   endif()
   unset(_DESIRED_HDF5_VERSION)
 
+  if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY)
+    list(APPEND _FIND_H5_ARGS QUIET)
+  endif()
+
+  if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED)
+    list(APPEND _FIND_H5_ARGS REQUIRED)
+  endif()
+
+  set(_FIND_H5_ARGS "${_FIND_H5_ARGS};COMPONENTS;C")
+
   # now we actually find the dependencies
-  find_package(HDF5 COMPONENTS C ${_EXTRA_ARGS})
+  find_package(HDF5 ${_FIND_H5_ARGS})
   if(NOT HDF5_FOUND)
     _FIND_GRACKLE_FAIL(
       "Grackle couldn't be found because dependency HDF5 couldn't be found")
   endif()
+  unset(_FIND_H5_ARGS)
   # declare HDF5 target (details are omitted that aren't needed for linking)
   add_library(GRACKLE_HDF5_C INTERFACE IMPORTED)
   set_target_properties(GRACKLE_HDF5_C PROPERTIES

cmake/installation_rules.cmake

@@ -147,7 +147,7 @@ if (BUILD_SHARED_LIBS)
   # install-rule to make a symlink called libgrackle.so to support compilation
   # with `-lgrackle`. (This is consistent with the classic build-system)
   install(CODE "
-    set(_prefix \"${CMAKE_INSTALL_PREFIX}\")
+    set(_prefix \"\${CMAKE_INSTALL_PREFIX}\")
     if(DEFINED ENV{DESTDIR})
       message(WARNING
         \"linking to libgrackle.so (during install) is untested with DESTDIR\")
@@ -186,19 +186,14 @@ else()
 endif()
 
 get_implicit_link_reqs(Fortran Fortran_implicit_libs Fortran_implicit_linkdirs)
-set(_TOOLCHAIN_LINK_LIBS ${Fortran_implicit_libs})
-
-# on most unix-like platforms (but not macOS), we need to explicitly link to
-# to the standard library's math functions
-# -> here we determine based on whether our custom toolchain::m target
-#    is a dummy placeholder or not whether to add this target
-get_target_property(toolchain_m_prop toolchain::m IMPORTED_LIBNAME)
-if(${toolchain_m_prop})
-  list(APPEND _TOOLCHAIN_LINK_LIBS m) # explicit c requirement (but may
-                                      # be a duplicate)
-endif()
+set(_TOOLCHAIN_LINK_LIBS "${Fortran_implicit_libs}")
+get_implicit_link_reqs(CXX CXX_implicit_libs CXX_implicit_linkdirs)
+list(APPEND _TOOLCHAIN_LINK_LIBS ${CXX_implicit_libs})
+
 
-list(REMOVE_DUPLICATES _TOOLCHAIN_LINK_LIBS)
+# we previously did this, but I don't think this is a great idea until after we stop
+# worying about implicit fortran link dependencies
+#list(REMOVE_DUPLICATES _TOOLCHAIN_LINK_LIBS)
 
 
 # Define the grackle.pc file
@@ -260,6 +255,7 @@ string(REPLACE
 
 
 set(_STATIC_EXTRA_LINK_DIRS ${Fortran_implicit_linkdirs})
+list(APPEND _STATIC_EXTRA_LINK_DIRS ${CXX_implicit_linkdirs})
 list(TRANSFORM _STATIC_EXTRA_LINK_DIRS PREPEND "-L")
 string(REPLACE
   ";" " " _STATIC_EXTRA_LINK_DIRS "${_STATIC_EXTRA_LINK_DIRS}")
@@ -298,7 +294,7 @@ else()
   # if shared library was previously installed, install grackle-conventional.pc
   # as grackle.pc. Otherwise, install grackle-static.pc as grackle.pc
   install(CODE "
-    set(_prefix \"${CMAKE_INSTALL_PREFIX}\")
+    set(_prefix \"\${CMAKE_INSTALL_PREFIX}\")
     if(DEFINED ENV{DESTDIR})
       message(WARNING
         \"linking to libgrackle.so (during install) is untested with DESTDIR\")

dependencies.cmake

@@ -75,18 +75,3 @@ if (GRACKLE_USE_OPENMP)
   endif()
   find_package(OpenMP REQUIRED COMPONENTS ${_GRACKLE_OMP_COMPONENTS})
 endif()
-
-# define target to link the math functions of the C standard library
-# (i.e. the -lm flag). This is commonly needed on unix-like platforms
-# -> For platforms that don't need libm, this target acts as a dummy
-#    placeholder (that does nothing)
-# -> The -lm flag should NOT be used on MacOS (while CMake is smart enough to
-#    not pass it to the linker, it will mess with exporting linker flags)
-#
-# NOTE: when we start using C++ in the core grackle library, we can remove
-# everything related to the toolchain::m variable (since the C++ runtime
-# library is ALWAYS linked to the math functions)
-add_library(toolchain::m INTERFACE IMPORTED)
-if (UNIX AND NOT CMAKE_SYSTEM_NAME STREQUAL "Darwin")
-  set_target_properties(toolchain::m PROPERTIES IMPORTED_LIBNAME "m")
-endif()

doc/source/CodeFormatting.rst

@@ -1,3 +1,5 @@
+.. _static_checks:
+
 Formatting, Linting, Other Checks
 =================================
 

doc/source/Contributing.rst

@@ -33,7 +33,7 @@ The design rationale for the CMake build-system tries to walk a fine line betwee
 
 The problem with the second philopsphy is that it assumes that the person building software from source is already well-versed in CMake (e.g. a developer, someone who will package and distribute your software, someone who wants to directly embed your software within their project), or they are a motivated developer/user who can be expected to learn CMake.
 It implicitly assumes that most users of a project won't ever need to directly build your software from source; they will instead install prebuilt and packaged copies of the software that are distributed through other channels (e.g. through package managers like apt, dnf, homebrew OR downloadable precompiled binaries through a website OR some kind of installer).
-While this implicit assumption may be accurate for most CMake software, it obviously doesn't apply to scientific software).
+While this implicit assumption may be accurate for most CMake software (or even most open-source software), it obviously doesn't apply to scientific software.
 
 It may be tempting to dismiss the second philosophy for scientific software.
 In fact, it doesn't provide substantial benefits in build-systems that simply build an application (like a simulation code).