Merge pull request #3961 from easybuilders/4.5.x

release EasyBuild v4.5.3

.github/workflows/container_tests.yml

@@ -0,0 +1,93 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for container support
+on: [push, pull_request]
+jobs:
+  build:
+    # stick to Ubuntu 18.04, where we can still easily install yum via 'apt-get install'
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python: [2.7, 3.6]
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{matrix.python}}
+        architecture: x64
+
+    - name: install OS & Python packages
+      run: |
+        # for building Singularity images
+        sudo apt-get install rpm
+        sudo apt-get install yum
+        # for modules tool
+        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
+        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
+        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
+            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+        fi
+
+    - name: install Lmod
+      run: |
+          # avoid downloading modules tool sources into easybuild-framework dir
+          cd $HOME
+          export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
+          # install Lmod
+          source $INSTALL_DEP Lmod-8.4.27 $HOME
+          # changes in environment are not passed to other steps, so need to create files...
+          echo $MOD_INIT > mod_init
+          echo $PATH > path
+          if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
+
+    # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
+    - name: install Singularity
+      run: |
+        # install alien, which can be used to convert RPMs to Debian packages
+        sudo apt-get install alien
+        alien --version
+        # determine latest version of Singularity available in EPEL, and download RPM
+        singularity_rpm=$(curl -sL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*singularity/singularity/g' | sed 's/rpm.*/rpm/g')
+        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/${singularity_rpm}
+        # convert Singularity RPM to Debian package, and install it
+        sudo alien -d ${singularity_rpm}
+        sudo apt install ./singularity*.deb
+        singularity --version
+
+    - name: install sources
+      run: |
+          # install from source distribution tarball, to test release as published on PyPI
+          python setup.py sdist
+          ls dist
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+          pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz
+
+    - name: run test
+      run: |
+          # run tests *outside* of checked out easybuild-framework directory,
+          # to ensure we're testing installed version (see previous step)
+          cd $HOME
+          # initialize environment for modules tool
+          if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
+          source $(cat $HOME/mod_init); type module
+          # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
+          # also pick up changes to $PATH set by sourcing $MOD_INIT
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          export PATH=$PREFIX/bin:$(cat $HOME/path)
+          export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
+          eb --version
+          # create $HOME/.rpmmacros, see also https://github.com/apptainer/singularity/issues/241
+          echo '%_var /var' > $HOME/.rpmmacros
+          echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros
+          # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild;
+          # see https://docs.easybuild.io/en/latest/Containers.html
+          curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
+          export EASYBUILD_CONTAINERPATH=$PWD
+          export EASYBUILD_CONTAINER_CONFIG='bootstrap=yum,osversion=7'
+          eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
+          singularity exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1)
+          singularity exec bzip2-1.0.8.sif bzip2 --help

.github/workflows/linting.yml

@@ -7,7 +7,7 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: set up Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v2
       with:
         python-version: 3.8
 

RELEASE_NOTES

@@ -4,6 +4,26 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.5.3 (February 11th 2022)
+---------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - also check for git in --check-eb-deps (#3954)
+  - add end2end test for 'eb --containerize' (#3958)
+- various bug fixes, including:
+  - take into account that patch files can also be zipped when checking filename extension for patches (#3936)
+  - initialize BACKUP_PKG_URL with empty string in install_eb_dep.sh script (#3939)
+  - fix get_os_name and get_os_version to avoid reporting UNKNOWN in output of eb --show-system-info (#3942)
+  - consistently use actions/setup-python@v2 in CI workflows (#3944)
+  - switch to using pip3 for installing EasyBuild in container recipes generated by EasyBuild (#3945)
+  - specify easybuild.io as EasyBuild homepage in setup.py (#3947)
+  - avoid crash in get_os_version on modern SLES-based OSs (#3955)
+- other changes:
+  - indicate compatibility with Python 3.10 in setup.py (#3940)
+
+
 v4.5.2 (January 24th 2022)
 --------------------------
 

easybuild/scripts/install_eb_dep.sh

@@ -16,6 +16,8 @@ PKG_VERSION="${PKG##*-}"
 CONFIG_OPTIONS=
 PRECONFIG_CMD=
 
+BACKUP_PKG_URL=
+
 if [ "$PKG_NAME" == 'modules' ]; then
     PKG_URL="http://prdownloads.sourceforge.net/modules/${PKG}.tar.gz"
     BACKUP_PKG_URL="https://sources.easybuild.io/e/EnvironmentModules/${PKG}.tar.gz"

easybuild/tools/config.py

@@ -134,7 +134,7 @@
 JOB_DEPS_TYPE_ABORT_ON_ERROR = 'abort_on_error'
 JOB_DEPS_TYPE_ALWAYS_RUN = 'always_run'
 
-DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:16.04'
+DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:20.04'
 DOCKER_BASE_IMAGE_CENTOS = 'centos:7'
 
 LOCAL_VAR_NAMING_CHECK_ERROR = 'error'

easybuild/tools/containers/docker.py

@@ -35,6 +35,7 @@
 from easybuild.tools.containers.base import ContainerGenerator
 from easybuild.tools.containers.utils import det_os_deps
 from easybuild.tools.filetools import remove_dir
+from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS
 from easybuild.tools.run import run_cmd
 
 
@@ -44,9 +45,9 @@
 """
 
 DOCKER_INSTALL_EASYBUILD = """\
-RUN pip install -U pip setuptools && \\
-    hash -r pip && \\
-    pip install -U easybuild
+RUN pip3 install -U pip setuptools && \\
+    hash -r pip3&& \\
+    pip3 install -U easybuild
 
 RUN mkdir /app && \\
     mkdir /scratch && \\
@@ -61,37 +62,43 @@
 
 RUN set -x && \\
     . /usr/share/lmod/lmod/init/sh && \\
-    eb %(eb_opts)s --installpath=/app/ --prefix=/scratch --tmpdir=/scratch/tmp
+    eb --robot %(eb_opts)s --installpath=/app/ --prefix=/scratch --tmpdir=/scratch/tmp
 
-RUN touch ${HOME}/.profile && \\
-    echo '\\n# Added by easybuild docker packaging' >> ${HOME}/.profile && \\
-    echo 'source /usr/share/lmod/lmod/init/bash' >> ${HOME}/.profile && \\
-    echo 'module use %(init_modulepath)s' >> ${HOME}/.profile && \\
-    echo 'module load %(mod_names)s' >> ${HOME}/.profile
+RUN touch ${HOME}/.bashrc && \\
+    echo '' >> ${HOME}/.bashrc && \\
+    echo '# Added by easybuild docker packaging' >> ${HOME}/.bashrc && \\
+    echo 'source /usr/share/lmod/lmod/init/bash' >> ${HOME}/.bashrc && \\
+    echo 'module use %(init_modulepath)s' >> ${HOME}/.bashrc && \\
+    echo 'module load %(mod_names)s' >> ${HOME}/.bashrc
 
 CMD ["/bin/bash", "-l"]
 """
 
-DOCKER_UBUNTU1604_INSTALL_DEPS = """\
+DOCKER_UBUNTU2004_INSTALL_DEPS = """\
 RUN apt-get update && \\
-    apt-get install -y python python-pip lmod curl wget
+    DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-pip lmod \\
+        curl wget git bzip2 gzip tar zip unzip xz-utils \\
+        patch automake git debianutils \\
+        g++ libdata-dump-perl libthread-queue-any-perl libssl-dev
 
 RUN OS_DEPS='%(os_deps)s' && \\
-    test -n "${OS_DEPS}" && \\
     for dep in ${OS_DEPS}; do apt-get -qq install ${dep} || true; done
 """
 
 DOCKER_CENTOS7_INSTALL_DEPS = """\
 RUN yum install -y epel-release && \\
-    yum install -y python python-pip Lmod curl wget git
+    yum install -y python3 python3-pip Lmod curl wget git \\
+        bzip2 gzip tar zip unzip xz \\
+        patch make git which \\
+        gcc-c++ perl-Data-Dumper perl-Thread-Queue openssl-dev
 
 RUN OS_DEPS='%(os_deps)s' && \\
     test -n "${OS_DEPS}" && \\
     yum --skip-broken install -y "${OS_DEPS}" || true
 """
 
 DOCKER_OS_INSTALL_DEPS_TMPLS = {
-    DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU1604_INSTALL_DEPS,
+    DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU2004_INSTALL_DEPS,
     DOCKER_BASE_IMAGE_CENTOS: DOCKER_CENTOS7_INSTALL_DEPS,
 }
 
@@ -125,9 +132,12 @@ def resolve_template_data(self):
 
         ec = self.easyconfigs[-1]['ec']
 
-        init_modulepath = os.path.join("/app/modules/all", *self.mns.det_init_modulepaths(ec))
+        # We are using the default MNS inside the container
+        docker_mns = EasyBuildMNS()
+
+        init_modulepath = os.path.join("/app/modules/all", *docker_mns.det_init_modulepaths(ec))
 
-        mod_names = [e['ec'].full_mod_name for e in self.easyconfigs]
+        mod_names = [docker_mns.det_full_module_name(e['ec']) for e in self.easyconfigs]
 
         eb_opts = [os.path.basename(e['spec']) for e in self.easyconfigs]
 

easybuild/tools/containers/singularity.py

@@ -272,9 +272,9 @@ def resolve_template_data(self):
                 # EPEL is required for installing Lmod & python-pip
                 'epel-release',
                 # EasyBuild requirements
-                'python setuptools Lmod',
-                # pip is used to install EasyBuild packages
-                'python-pip',
+                'python3 setuptools Lmod',
+                # pip3 is used to install EasyBuild packages
+                'python3-pip',
                 # useful utilities
                 'bzip2 gzip tar zip unzip xz',  # extracting sources
                 'curl wget',  # downloading
@@ -308,13 +308,13 @@ def resolve_template_data(self):
         template_data['install_os_deps'] = '\n'.join(install_os_deps)
 
         # install (latest) EasyBuild in container image
-        # use 'pip install', unless custom commands are specified via 'install_eb' keyword
+        # use 'pip3 install', unless custom commands are specified via 'install_eb' keyword
         if 'install_eb' not in template_data:
             template_data['install_eb'] = '\n'.join([
-                "# install EasyBuild using pip",
+                "# install EasyBuild using pip3",
                 # upgrade pip
-                "pip install -U pip",
-                "pip install easybuild",
+                "pip3 install -U pip",
+                "pip3 install easybuild",
             ])
 
         # if no custom value is specified for 'post_commands' keyword,

easybuild/tools/filetools.py

@@ -163,6 +163,8 @@
     '.sh': "cp -a %(filepath)s .",
 }
 
+ZIPPED_PATCH_EXTS = ('.bz2', '.gz', '.xz')
+
 # global set of names of locks that were created in this session
 global_lock_names = set()
 
@@ -1493,8 +1495,11 @@ def create_patch_info(patch_spec):
                                  str(patch_spec))
 
     elif isinstance(patch_spec, string_type):
-        if not patch_spec.endswith('.patch'):
-            _log.deprecated("Use of patch file with filename that doesn't end with .patch: %s" % patch_spec, '5.0')
+        allowed_patch_exts = ['.patch' + x for x in ('',) + ZIPPED_PATCH_EXTS]
+        if not any(patch_spec.endswith(x) for x in allowed_patch_exts):
+            msg = "Use of patch file with filename that doesn't end with correct extension: %s " % patch_spec
+            msg += "(should be any of: %s)" % (', '.join(allowed_patch_exts))
+            _log.deprecated(msg, '5.0')
         patch_info = {'name': patch_spec}
     else:
         error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s"
@@ -1548,7 +1553,7 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa
     # split in stem (filename w/o extension) + extension
     patch_stem, patch_extension = os.path.splitext(os.path.split(abs_patch_file)[1])
     # Supports only bz2, gz and xz. zip can be archives which are not supported.
-    if patch_extension in ['.gz', '.bz2', '.xz']:
+    if patch_extension in ZIPPED_PATCH_EXTS:
         # split again to get the second extension
         patch_subextension = os.path.splitext(patch_stem)[1]
         if patch_subextension == ".patch":

easybuild/tools/systemtools.py

@@ -168,6 +168,7 @@
     '7z': "extracting sources (.iso)",
     'bunzip2': "decompressing sources (.bz2, .tbz, .tbz2, ...)",
     DPKG: "checking OS dependencies (Debian, Ubuntu, ...)",
+    'git': "downloading sources using 'git clone'",
     'gunzip': "decompressing source files (.gz, .tgz, ...)",
     'make': "build tool",
     'patch': "applying patch files",
@@ -697,9 +698,6 @@ def get_os_name():
         res = name_regex.search(os_release_txt)
         if res:
             os_name = res.group('name')
-    else:
-        # no easy way to determine name of Linux distribution
-        os_name = None
 
     os_name_map = {
         'red hat enterprise linux server': 'RHEL',
@@ -742,44 +740,33 @@ def get_os_version():
             res = version_regex.search(os_release_txt)
             if res:
                 os_version = res.group('version')
-    else:
-        os_version = None
 
     if os_version:
-        if get_os_name() in ["suse", "SLES"]:
-
-            # SLES subversions can only be told apart based on kernel version,
-            # see http://wiki.novell.com/index.php/Kernel_versions
-            version_suffixes = {
-                '11': [
-                    ('2.6.27', ''),
-                    ('2.6.32', '_SP1'),
-                    ('3.0.101-63', '_SP4'),
-                    # not 100% correct, since early SP3 had 3.0.76 - 3.0.93, but close enough?
-                    ('3.0.101', '_SP3'),
-                    # SP2 kernel versions range from 3.0.13 - 3.0.101
-                    ('3.0', '_SP2'),
-                ],
-
-                '12': [
-                    ('3.12.28', ''),
-                    ('3.12.49', '_SP1'),
-                ],
-            }
-
+        # older SLES subversions can only be told apart based on kernel version,
+        # see http://wiki.novell.com/index.php/Kernel_versions
+        sles_version_suffixes = {
+            '11': [
+                ('2.6.27', ''),
+                ('2.6.32', '_SP1'),
+                ('3.0.101-63', '_SP4'),
+                # not 100% correct, since early SP3 had 3.0.76 - 3.0.93, but close enough?
+                ('3.0.101', '_SP3'),
+                # SP2 kernel versions range from 3.0.13 - 3.0.101
+                ('3.0', '_SP2'),
+            ],
+
+            '12': [
+                ('3.12.28', ''),
+                ('3.12.49', '_SP1'),
+            ],
+        }
+        if get_os_name() in ['suse', 'SLES'] and os_version in sles_version_suffixes:
             # append suitable suffix to system version
-            if os_version in version_suffixes.keys():
-                kernel_version = platform.uname()[2]
-                known_sp = False
-                for (kver, suff) in version_suffixes[os_version]:
-                    if kernel_version.startswith(kver):
-                        os_version += suff
-                        known_sp = True
-                        break
-                if not known_sp:
-                    suff = '_UNKNOWN_SP'
-            else:
-                raise EasyBuildError("Don't know how to determine subversions for SLES %s", os_version)
+            kernel_version = platform.uname()[2]
+            for (kver, suff) in sles_version_suffixes[os_version]:
+                if kernel_version.startswith(kver):
+                    os_version += suff
+                    break
 
         return os_version
     else:

easybuild/tools/version.py

@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.5.2')
+VERSION = LooseVersion('4.5.3')
 UNKNOWN = 'UNKNOWN'