Merge pull request #3988 from easybuilders/4.5.x

release EasyBuild v4.5.4

.github/workflows/bootstrap_script.yml

@@ -107,7 +107,7 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20210715.01 784dd29063d941be2d8b70e4c2eec12a9afe360f3ef8f753dcb518abf43ca7de"
+          EB_BOOTSTRAP_EXPECTED="20210715.01 0ffdc17ed7eacf78369c9cd6743728f36e61bb8bf5c1bdc1e23cf2040b1ce301"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
           # test bootstrap script

.github/workflows/container_tests.yml

@@ -20,6 +20,8 @@ jobs:
 
     - name: install OS & Python packages
       run: |
+        # ensure package list is up to date to avoid 404's for old packages
+        sudo apt-get update -yqq
         # for building Singularity images
         sudo apt-get install rpm
         sudo apt-get install yum

.github/workflows/linting.yml

@@ -3,13 +3,17 @@ on: [push, pull_request]
 jobs:
   python-linting:
     runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']
+
     steps:
     - uses: actions/checkout@v2
 
     - name: set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: ${{ matrix.python-version }}
 
     - name: install Python packages
       run: |
@@ -18,4 +22,10 @@ jobs:
 
     - name: Run flake8 to verify PEP8-compliance of Python code
       run: |
-        flake8
+        # don't check py2vs3/py3.py when testing with Python 2, and vice versa
+        if [[ "${{ matrix.python-version }}" =~ "2." ]]; then
+            py_excl=py3
+        else
+            py_excl=py2
+        fi
+        flake8 --exclude ./easybuild/tools/py2vs3/${py_excl}.py

RELEASE_NOTES

@@ -4,6 +4,29 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.5.4 (March 31st 2022)
+------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - warn about potentially missing patches in --new-pr (#3759, #3966)
+  - add support for 'clone_into' field in git_config source spec to specify different name for top-level directory (#3949)
+  - add bash completion for easyconfigs from local dir but not robot search path (#3953)
+  - add a 'sync pr' message when the PR has a mergeable state but is showing a failed status for the test suite on the last commit (#3967)
+  - add gmpit toolchain definition (GCC + MPItrampoline) (#3971)
+  - use 'zypper search -i' to check whether specified OS dependency is installed on openSUSE + make sure that rpm is considered for checking OS dependencies on RHEL8 (#3973)
+  - add support for post-install patches (#3974)
+  - add support for 'download_instructions' easyconfig parameter key to specify some download or installation steps for user in case of complicated way of obtaining needed files (#3976, #3981)
+  - also try collecting AMD GPU info (via rocm-smi) for --show-system-info (#3978, #3982)
+- various bug fixes, including:
+  - ensure --review-pr can find dependencies included in PR (#3979)
+  - run 'apt-get update' in GitHub Actions workflow for container tests to update container package list before installing anything (#3985)
+- other changes:
+  - enable code linting check for all supported Python versions (#3725)
+  - update copyright lines for 2022 (#3986)
+
+
 v4.5.3 (February 11th 2022)
 ---------------------------
 

easybuild/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2011-2021 Ghent University
+# Copyright 2011-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/exceptions.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2015-2021 Ghent University
+# Copyright 2015-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/fancylogger.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2021 Ghent University
+# Copyright 2011-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/generaloption.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2021 Ghent University
+# Copyright 2011-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/testing.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2014-2021 Ghent University
+# Copyright 2014-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2009-2021 Ghent University
+# Copyright 2009-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyblock.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2021 Ghent University
+# Copyright 2009-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -364,7 +364,7 @@ def get_checksum_for(self, checksums, filename=None, index=None):
         else:
             raise EasyBuildError("Invalid type for checksums (%s), should be list, tuple or None.", type(checksums))
 
-    def fetch_source(self, source, checksum=None, extension=False):
+    def fetch_source(self, source, checksum=None, extension=False, download_instructions=None):
         """
         Get a specific source (tarball, iso, url)
         Will be tested for existence or can be located
@@ -400,7 +400,8 @@ def fetch_source(self, source, checksum=None, extension=False):
         # check if the sources can be located
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
         path = self.obtain_file(filename, extension=extension, download_filename=download_filename,
-                                force_download=force_download, urls=source_urls, git_config=git_config)
+                                force_download=force_download, urls=source_urls, git_config=git_config,
+                                download_instructions=download_instructions)
         if path is None:
             raise EasyBuildError('No file found for source %s', filename)
 
@@ -454,13 +455,17 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
         Add a list of patches.
         All patches will be checked if a file exists (or can be located)
         """
+        post_install_patches = []
         if patch_specs is None:
-            patch_specs = self.cfg['patches']
+            # if no patch_specs are specified, use all pre-install and post-install patches
+            post_install_patches = self.cfg['postinstallpatches']
+            patch_specs = self.cfg['patches'] + post_install_patches
 
         patches = []
         for index, patch_spec in enumerate(patch_specs):
 
             patch_info = create_patch_info(patch_spec)
+            patch_info['postinstall'] = patch_spec in post_install_patches
 
             force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES]
             path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download)
@@ -551,6 +556,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                     source_urls = ext_options.get('source_urls', [])
                     checksums = ext_options.get('checksums', [])
 
+                    download_instructions = ext_options.get('download_instructions')
+
                     if ext_options.get('nosource', None):
                         self.log.debug("No sources for extension %s, as indicated by 'nosource'", ext_name)
 
@@ -582,7 +589,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                             source['source_urls'] = source_urls
 
                         if fetch_files:
-                            src = self.fetch_source(source, checksums, extension=True)
+                            src = self.fetch_source(source, checksums, extension=True,
+                                                    download_instructions=download_instructions)
                             ext_src.update({
                                 # keep track of custom extract command (if any)
                                 'extract_cmd': src['cmd'],
@@ -604,7 +612,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
 
                         if fetch_files:
                             src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
-                                                        force_download=force_download)
+                                                        force_download=force_download,
+                                                        download_instructions=download_instructions)
                             if src_path:
                                 ext_src.update({'src': src_path})
                             else:
@@ -678,7 +687,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None):
+                    git_config=None, download_instructions=None):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -865,8 +874,19 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     self.dry_run_msg("  * %s (MISSING)", filename)
                     return filename
                 else:
-                    raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... "
-                                         "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                    error_msg = "Couldn't find file %s anywhere, "
+                    if download_instructions is None:
+                        download_instructions = self.cfg['download_instructions']
+                    if download_instructions is not None and download_instructions != "":
+                        msg = "\nDownload instructions:\n\n" + download_instructions + '\n'
+                        print_msg(msg, prefix=False, stderr=True)
+                        error_msg += "please follow the download instructions above, and make the file available "
+                        error_msg += "in the active source path (%s)" % ':'.join(source_paths())
+                    else:
+                        error_msg += "and downloading it didn't work either... "
+                        error_msg += "Paths attempted (in order): %s " % ', '.join(failedpaths)
+
+                    raise EasyBuildError(error_msg, filename)
 
     #
     # GETTER/SETTER UTILITY FUNCTIONS
@@ -1485,12 +1505,11 @@ def make_module_req(self):
                 if key in keys_requiring_files:
                     # only retain paths that contain at least one file
                     recursive = keys_requiring_files[key]
-                    retained_paths = [
-                        path
-                        for path, fullpath in ((path, os.path.join(self.installdir, path)) for path in paths)
-                        if os.path.isdir(fullpath)
-                        and dir_contains_files(fullpath, recursive=recursive)
-                    ]
+                    retained_paths = []
+                    for pth in paths:
+                        fullpath = os.path.join(self.installdir, pth)
+                        if os.path.isdir(fullpath) and dir_contains_files(fullpath, recursive=recursive):
+                            retained_paths.append(pth)
                     if retained_paths != paths:
                         self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s",
                                       key, paths, retained_paths)
@@ -2231,7 +2250,7 @@ def fetch_step(self, skip_checksums=False):
             self.dry_run_msg("\nList of patches:")
 
         # fetch patches
-        if self.cfg['patches']:
+        if self.cfg['patches'] + self.cfg['postinstallpatches']:
             if isinstance(self.cfg['checksums'], (list, tuple)):
                 # if checksums are provided as a list, first entries are assumed to be for sources
                 patches_checksums = self.cfg['checksums'][len(self.cfg['sources']):]
@@ -2403,11 +2422,15 @@ def extract_step(self):
             else:
                 raise EasyBuildError("Unpacking source %s failed", src['name'])
 
-    def patch_step(self, beginpath=None):
+    def patch_step(self, beginpath=None, patches=None):
         """
         Apply the patches
         """
-        for patch in self.patches:
+        if patches is None:
+            # if no patches are specified, use all non-post-install patches
+            patches = [p for p in self.patches if not p['postinstall']]
+
+        for patch in patches:
             self.log.info("Applying patch %s" % patch['name'])
             trace_msg("applying patch %s" % patch['name'])
 
@@ -2822,13 +2845,25 @@ def run_post_install_commands(self, commands=None):
                     raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd)
                 run_cmd(cmd, simple=True, log_ok=True, log_all=True)
 
+    def apply_post_install_patches(self, patches=None):
+        """
+        Apply post-install patch files that are specified via the 'postinstallpatches' easyconfig parameter.
+        """
+        if patches is None:
+            patches = [p for p in self.patches if p['postinstall']]
+
+        self.log.debug("Post-install patches to apply: %s", patches)
+        if patches:
+            self.patch_step(beginpath=self.installdir, patches=patches)
+
     def post_install_step(self):
         """
         Do some postprocessing
         - run post install commands if any were specified
         """
 
         self.run_post_install_commands()
+        self.apply_post_install_patches()
 
         self.fix_shebang()
 

easybuild/framework/easyconfig/__init__.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2021 Ghent University
+# Copyright 2009-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/constants.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2013-2021 Ghent University
+# Copyright 2013-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/default.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2021 Ghent University
+# Copyright 2009-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -90,6 +90,7 @@
     'checksums': [[], "Checksums for sources and patches", BUILD],
     'configopts': ['', 'Extra options passed to configure (default already has --prefix)', BUILD],
     'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with (if supported)", BUILD],
+    'download_instructions': ['', "Specify steps to aquire necessary file, if obtaining it is difficult", BUILD],
     'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected "
                         "based on the software name", BUILD],
     'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD],
@@ -114,6 +115,7 @@
     'preinstallopts': ['', 'Extra prefix options for installation.', BUILD],
     'pretestopts': ['', 'Extra prefix options for test.', BUILD],
     'postinstallcmds': [[], 'Commands to run after the install step.', BUILD],
+    'postinstallpatches': [[], 'Patch files to apply after running the install step.', BUILD],
     'required_linked_shared_libs': [[], "List of shared libraries (names, file names, or paths) which must be "
                                         "linked in all installed binaries/libraries", BUILD],
     'runtest': [None, ('Indicates if a test should be run after make; should specify argument '

easybuild/framework/easyconfig/easyconfig.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2021 Ghent University
+# Copyright 2009-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/__init__.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2021 Ghent University
+# Copyright 2013-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/convert.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2014-2021 Ghent University
+# Copyright 2014-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/format.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2021 Ghent University
+# Copyright 2013-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/one.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2021 Ghent University
+# Copyright 2013-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/pyheaderconfigobj.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2021 Ghent University
+# Copyright 2013-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/two.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2021 Ghent University
+# Copyright 2013-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/version.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2021 Ghent University
+# Copyright 2013-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/format/yeb.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2021 Ghent University
+# Copyright 2013-2022 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),