diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
index 7327d876ba..e74b65cc79 100644
--- a/.github/workflows/end2end.yml
+++ b/.github/workflows/end2end.yml
@@ -18,6 +18,7 @@ jobs:
fail-fast: false
container:
image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+ env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
steps:
- name: Check out the repo
uses: actions/checkout@v3
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
index c9f42891ec..8a8b13af96 100644
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -101,12 +101,12 @@ jobs:
# and are only run after the PR gets merged
GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
run: |
- # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
+ # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit
# tests that require a GitHub token are skipped automatically when no GitHub token is available
if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
if [ ! -z $GITHUB_TOKEN ]; then
- SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
- python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+ SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"
+ python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"
fi
echo "GitHub token installed!"
else
@@ -191,7 +191,17 @@ jobs:
# run test suite
python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
# try and make sure output of running tests is clean (no printed messages/warnings)
- IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
+ IGNORE_PATTERNS="no GitHub token available"
+ IGNORE_PATTERNS+="|skipping SvnRepository test"
+ IGNORE_PATTERNS+="|requires Lmod as modules tool"
+ IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device"
+ IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]"
+ IGNORE_PATTERNS+="|from cryptography.* import "
+ IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2"
+ IGNORE_PATTERNS+="|Blowfish"
+ IGNORE_PATTERNS+="|GC3Pie not available, skipping test"
+ IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved"
+ IGNORE_PATTERNS+="|algorithms.TripleDES"
# '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml
index 1b921ee83c..aa00f9a2fd 100644
--- a/.github/workflows/unit_tests_python2.yml
+++ b/.github/workflows/unit_tests_python2.yml
@@ -16,6 +16,7 @@ jobs:
# CentOS 7.9 container that already includes Lmod & co,
# see https://github.com/easybuilders/easybuild-containers
image: ghcr.io/easybuilders/centos-7.9-amd64
+ env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
steps:
- uses: actions/checkout@v3
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 5a45be8a56..7f7ade0c1e 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -4,6 +4,43 @@ For more detailed information, please see the git log.
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
+v4.9.3 (14 September 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+ - add support for `--extra-source-urls` to fetch sources from additional URLs (#4079)
+ - add definition for gmpflf toolchain (#4566, #4571)
+ - reuse pre-computed checksums (#4569)
+ - add `cuda_cc_space_sep` variant that does not have periods (#4583)
+ - add `--skip-sanity-check` option (#4590)
+ - add `GNU_FTP_SOURCE` template constant (#4597)
+ - improve error messages for empty easyconfigs (#4603)
+ - improve help string for `--dep-graph` (#4610)
+ - only call `_sanity_check_step_extensions` if `--skip-extensions` is not set (#4620)
+ - add support for `--software-commit` and an associated template `%(software_commit)s` (#4628)
+- various bug fixes, including:
+ - correctly evaluate result for `--dep-graph` (#4554)
+ - fix fetch progress bar showing to many files (#4568)
+ - resolve internal for imkl>=2021 version subdir via "latest" symlink (#4570)
+ - fix typo in message about including an easyblock from a commit (#4575)
+ - don't use special flags for `strict`, `precise`, `loose`, `veryloose` toolchain options on RISC-V (#4576)
+ - fix help text for `cuda_compute_capabilities` template (#4589)
+ - fix help message for `--http-headers-fields-urlpat` configuration option (#4594)
+ - fix `test_compiler_cache` in case `gcc` is available multiple times (#4599)
+ - handle post-install patches in check_checksums_for (#4605)
+ - fix `copy_file` with a folder as the target (#4609)
+ - allow for case where `homepage = None` when generating the docs (#4626)
+ - fix test_github_det_commit_status by using more recent commits (#4636)
+- other changes:
+ - clean up code that was only there to support Python 2.6 + avoid syntax warnings when parsing py2vs3/py.p2 with Python 3.x (#3788)
+ - use Intel's oneAPI Fortran compiler by default for version 2024.0.0 and newer (`oneapi_fortran` toolchain option set to `True`) (#4567)
+ - allow using Node 16 actions in CI (#4574)
+ - remove a superflous check in `EasyBlock.run_all_steps` (#4623)
+ - remove trailing dots from backup message produced by --inject-checksums (#4632)
+
+
v4.9.2 (12 June 2024)
---------------------
diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py
index 90248fb0cd..c3660f7cbf 100644
--- a/easybuild/base/generaloption.py
+++ b/easybuild/base/generaloption.py
@@ -97,11 +97,16 @@ def what_str_list_tuple(name):
"""Given name, return separator, class and helptext wrt separator.
(Currently supports strlist, strtuple, pathlist, pathtuple)
"""
- sep = ','
- helpsep = 'comma'
if name.startswith('path'):
sep = os.pathsep
helpsep = 'pathsep'
+ elif name.startswith('url'):
+ # | is one of the only characters not in the grammar for URIs (RFC3986)
+ sep = '|'
+ helpsep = '|'
+ else:
+ sep = ','
+ helpsep = 'comma'
klass = None
if name.endswith('list'):
@@ -182,6 +187,7 @@ class ExtOption(CompleterOption):
- strlist, strtuple : convert comma-separated string in a list resp. tuple of strings
- pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings
- the path separator is OS-dependent
+ - urllist, urltuple: convert string seperated by '|' to a list resp. tuple of strings
"""
EXTEND_SEPARATOR = ','
@@ -198,7 +204,7 @@ class ExtOption(CompleterOption):
TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR
ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS
- TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']]
+ TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path', 'url']]
TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST}
TYPE_CHECKER.update(Option.TYPE_CHECKER)
TYPES = tuple(TYPE_STRLIST + list(Option.TYPES))
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 80cddf8819..0b76f5bff2 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -72,6 +72,7 @@
from easybuild.tools.build_log import print_error, print_msg, print_warning
from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
+from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa
from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
from easybuild.tools.config import install_path, log_path, package_path, source_paths
from easybuild.tools.environment import restore_env, sanitize_env
@@ -105,9 +106,6 @@
from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
-
-EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io'
-
DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64')
MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP]
@@ -674,14 +672,16 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
src_fn = os.path.basename(src_path)
# report both MD5 and SHA256 checksums, since both are valid default checksum types
+ src_checksums = {}
for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
+ src_checksums[checksum_type] = src_checksum
self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
# verify checksum (if provided)
self.log.debug('Verifying checksums for extension source...')
fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0)
- if verify_checksum(src_path, fn_checksum):
+ if verify_checksum(src_path, fn_checksum, src_checksums):
self.log.info('Checksum for extension source %s verified', src_fn)
elif build_option('ignore_checksums'):
print_warning("Ignoring failing checksum verification for %s" % src_fn)
@@ -700,12 +700,15 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
ext_src.update({'patches': ext_patches})
if verify_checksums:
+ computed_checksums = {}
for patch in ext_patches:
patch = patch['path']
+ computed_checksums[patch] = {}
# report both MD5 and SHA256 checksums,
# since both are valid default checksum types
for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
checksum = compute_checksum(patch, checksum_type=checksum_type)
+ computed_checksums[patch][checksum_type] = checksum
self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
# verify checksum (if provided)
@@ -715,7 +718,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
patch_fn = os.path.basename(patch)
checksum = self.get_checksum_for(checksums, filename=patch_fn, index=idx+1)
- if verify_checksum(patch, checksum):
+ if verify_checksum(patch, checksum, computed_checksums[patch]):
self.log.info('Checksum for extension patch %s verified', patch_fn)
elif build_option('ignore_checksums'):
print_warning("Ignoring failing checksum verification for %s" % patch_fn)
@@ -754,7 +757,9 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
"""
srcpaths = source_paths()
- update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
+ # We don't account for the checksums file in the progress bar
+ if filename != 'checksum.json':
+ update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
if alt_location is None:
location = self.name
@@ -890,8 +895,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
source_urls = []
source_urls.extend(self.cfg['source_urls'])
- # add https://sources.easybuild.io as fallback source URL
- source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location))
+ # Add additional URLs as configured.
+ for url in build_option("extra_source_urls"):
+ url += "/" + name_letter + "/" + location
+ source_urls.append(url)
mkdir(targetdir, parents=True)
@@ -2475,7 +2482,7 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
checksum_issues = []
sources = ent.get('sources', [])
- patches = ent.get('patches', [])
+ patches = ent.get('patches', []) + ent.get('postinstallpatches', [])
checksums = ent.get('checksums', [])
# Single source should be re-wrapped as a list, and checksums with it
if isinstance(sources, dict):
@@ -3482,10 +3489,7 @@ def _sanity_check_step_extensions(self):
"""Sanity check on extensions (if any)."""
failed_exts = []
- if build_option('skip_extensions'):
- self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...")
- return
- elif not self.ext_instances:
+ if not self.ext_instances:
# class instances for extensions may not be initialized yet here,
# for example when using --module-only or --sanity-check-only
self.prepare_for_extensions()
@@ -3637,7 +3641,10 @@ def xs2str(xs):
# also run sanity check for extensions (unless we are an extension ourselves)
if not extension:
- self._sanity_check_step_extensions()
+ if build_option('skip_extensions'):
+ self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...")
+ else:
+ self._sanity_check_step_extensions()
linked_shared_lib_fails = self.sanity_check_linked_shared_libs()
if linked_shared_lib_fails:
@@ -3894,12 +3901,13 @@ def update_config_template_run_step(self):
self.cfg.generate_template_values()
def skip_step(self, step, skippable):
- """Dedice whether or not to skip the specified step."""
+ """Decide whether or not to skip the specified step."""
skip = False
force = build_option('force')
module_only = build_option('module_only') or self.cfg['module_only']
sanity_check_only = build_option('sanity_check_only')
skip_extensions = build_option('skip_extensions')
+ skip_sanity_check = build_option('skip_sanity_check')
skip_test_step = build_option('skip_test_step')
skipsteps = self.cfg['skipsteps']
@@ -3927,6 +3935,10 @@ def skip_step(self, step, skippable):
self.log.info("Skipping %s step because of sanity-check-only mode", step)
skip = True
+ elif skip_sanity_check and step == SANITYCHECK_STEP:
+ self.log.info("Skipping %s step as request via skip-sanity-check", step)
+ skip = True
+
elif skip_extensions and step == EXTENSIONS_STEP:
self.log.info("Skipping %s step as requested via skip-extensions", step)
skip = True
@@ -3937,9 +3949,9 @@ def skip_step(self, step, skippable):
else:
msg = "Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s, "
- msg += "sanity_check_only: %s, skip_extensions: %s, skip_test_step: %s)"
+ msg += "sanity_check_only: %s, skip_extensions: %s, skip_test_step: %s, skip_sanity_check: %s)"
self.log.debug(msg, step, skippable, self.skip, skipsteps, module_only, force,
- sanity_check_only, skip_extensions, skip_test_step)
+ sanity_check_only, skip_extensions, skip_test_step, skip_sanity_check)
return skip
@@ -4089,7 +4101,7 @@ def run_all_steps(self, run_test_cases):
Build and install this software.
run_test_cases (bool): run tests after building (e.g.: make test)
"""
- if self.cfg['stop'] and self.cfg['stop'] == 'cfg':
+ if self.cfg['stop'] == 'cfg':
return True
steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
@@ -4708,7 +4720,7 @@ def make_checksum_lines(checksums, indent_level):
# back up easyconfig file before injecting checksums
ec_backup = back_up_file(ec['spec'])
- print_msg("backup of easyconfig file saved to %s..." % ec_backup, log=_log)
+ print_msg("backup of easyconfig file saved to %s" % ec_backup, log=_log)
# compute & inject checksums for sources/patches
print_msg("injecting %s checksums for sources & patches in %s..." % (checksum_type, ec_fn), log=_log)
diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index 2b6aaa751a..a8c9a488b9 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -46,6 +46,7 @@
import functools
import os
import re
+from collections import OrderedDict
from contextlib import contextmanager
import easybuild.tools.filetools as filetools
@@ -74,7 +75,7 @@
from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
from easybuild.tools.modules import modules_tool, NoModulesTool
-from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
+from easybuild.tools.py2vs3 import create_base_metaclass, string_type
from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA
@@ -455,6 +456,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
self.path = path
self.rawtxt = read_file(path)
self.log.debug("Raw contents from supplied easyconfig file %s: %s", path, self.rawtxt)
+ if not self.rawtxt.strip():
+ raise EasyBuildError('Easyconfig file is empty')
else:
self.rawtxt = rawtxt
self.log.debug("Supplied raw easyconfig contents: %s" % self.rawtxt)
@@ -1909,7 +1912,9 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
else:
# if no easyblock specified, try to find if one exists
if name is None:
- name = "UNKNOWN"
+ if error_on_missing_easyblock:
+ raise EasyBuildError("No easyblock found as neither name nor easyblock were specified")
+ return None
# The following is a generic way to calculate unique class names for any funny software title
class_name = encode_class_name(name)
# modulepath will be the namespace + encoded modulename (from the classname)
diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py
index 9da6e9a2b9..be9bb28bd3 100644
--- a/easybuild/framework/easyconfig/templates.py
+++ b/easybuild/framework/easyconfig/templates.py
@@ -90,16 +90,19 @@
# template values which are only generated dynamically
TEMPLATE_NAMES_DYNAMIC = [
('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"),
- ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
- "as specify by the --sysroot configuration option"),
- ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"),
('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via "
"--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"),
('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"),
('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"),
+ ('cuda_cc_space_sep_no_period',
+ "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90')."),
('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"),
('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"),
+ ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"),
+ ('software_commit', "Git commit id to use for the software as specified by --software-commit command line option"),
+ ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
+ "as specified by the --sysroot configuration option"),
]
# constant templates that can be used in easyconfigs
@@ -128,7 +131,9 @@
('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
'download.savannah.gnu.org source url'),
('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s',
- 'gnu.org source url'),
+ 'gnu.org source url (ftp mirror)'),
+ ('GNU_FTP_SOURCE', 'https://ftp.gnu.org/gnu/%(namelower)s',
+ 'gnu.org source url (main ftp)'),
('GOOGLECODE_SOURCE', 'http://%(namelower)s.googlecode.com/files',
'googlecode.com source url'),
('LAUNCHPAD_SOURCE', 'https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/',
@@ -206,6 +211,9 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
# set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback
template_values['sysroot'] = build_option('sysroot') or ''
+ # set 'software_commit' template based on 'software_commit' configuration option, default to None
+ template_values['software_commit'] = build_option('software_commit') or ''
+
# step 1: add TEMPLATE_NAMES_EASYCONFIG
for name in TEMPLATE_NAMES_EASYCONFIG:
if name in ignore:
@@ -363,13 +371,14 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
# step 6. CUDA compute capabilities
# Use the commandline / easybuild config option if given, else use the value from the EC (as a default)
- cuda_compute_capabilities = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities')
- if cuda_compute_capabilities:
- template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities)
- template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities)
- template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities)
- template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities)
- sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities]
+ cuda_cc = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities')
+ if cuda_cc:
+ template_values['cuda_compute_capabilities'] = ','.join(cuda_cc)
+ template_values['cuda_cc_space_sep'] = ' '.join(cuda_cc)
+ template_values['cuda_cc_space_sep_no_period'] = ' '.join(cc.replace('.', '') for cc in cuda_cc)
+ template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_cc)
+ template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_cc)
+ sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_cc]
template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
template_values['cuda_sm_space_sep'] = ' '.join(sm_values)
diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
index 6f2a2741ad..cb92b36b76 100644
--- a/easybuild/framework/easyconfig/tools.py
+++ b/easybuild/framework/easyconfig/tools.py
@@ -45,6 +45,7 @@
import re
import sys
import tempfile
+from collections import OrderedDict
from easybuild.base import fancylogger
from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
@@ -54,7 +55,7 @@
from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars
from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
+from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning
from easybuild.tools.config import build_option
from easybuild.tools.environment import restore_env
from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
@@ -64,7 +65,6 @@
from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data
from easybuild.tools.github import fetch_files_from_commit, fetch_files_from_pr
from easybuild.tools.multidiff import multidiff
-from easybuild.tools.py2vs3 import OrderedDict
from easybuild.tools.toolchain.toolchain import is_system_toolchain
from easybuild.tools.toolchain.utilities import search_toolchain
from easybuild.tools.utilities import only_if_module_is_available, quote_str
@@ -219,10 +219,12 @@ def mk_node_name(spec):
if dep in spec['ec'].build_dependencies:
dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs)
- _dep_graph_dump(dgr, filename)
-
- if not build_option('silent'):
- print("Wrote dependency graph for %d easyconfigs to %s" % (len(specs), filename))
+ what = "dependency graph for %d easyconfigs to %s" % (len(specs), filename)
+ silent = build_option('silent')
+ if _dep_graph_dump(dgr, filename):
+ print_msg("Wrote " + what, silent=silent)
+ else:
+ print_error("Failed writing " + what, silent=silent)
@only_if_module_is_available('pygraph.readwrite.dot', pkgname='python-graph-dot')
@@ -232,9 +234,15 @@ def _dep_graph_dump(dgr, filename):
dottxt = dot.write(dgr)
if os.path.splitext(filename)[-1] == '.dot':
# create .dot file
- write_file(filename, dottxt)
+ try:
+ write_file(filename, dottxt)
+ except EasyBuildError as e:
+ print(str(e))
+ return False
+ else:
+ return True
else:
- _dep_graph_gv(dottxt, filename)
+ return _dep_graph_gv(dottxt, filename)
@only_if_module_is_available('gv', pkgname='graphviz-python')
@@ -242,8 +250,8 @@ def _dep_graph_gv(dottxt, filename):
"""Render dependency graph to file using graphviz."""
# try and render graph in specified file format
gvv = gv.readstring(dottxt)
- gv.layout(gvv, 'dot')
- gv.render(gvv, os.path.splitext(filename)[-1], filename)
+ if gv.layout(gvv, 'dot') is not False:
+ return gv.render(gvv, os.path.splitext(filename)[-1], filename)
def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None):
diff --git a/easybuild/main.py b/easybuild/main.py
index 451152f322..b6adc08559 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -727,9 +727,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr
if options.ignore_test_failure:
raise EasyBuildError("Found both ignore-test-failure and skip-test-step enabled. "
"Please use only one of them.")
- else:
- print_warning("Will not run the test step as requested via skip-test-step. "
- "Consider using ignore-test-failure instead and verify the results afterwards")
+ print_warning("Will not run the test step as requested via skip-test-step. "
+ "Consider using ignore-test-failure instead and verify the results afterwards")
+ if options.skip_sanity_check and options.sanity_check_only:
+ raise EasyBuildError("Found both skip-sanity-check and sanity-check-only enabled. "
+ "Please use only one of them.")
# if EasyStack file is provided, parse it, and loop over the items in the EasyStack file
if options.easystack:
diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py
index 548ac41187..e9748f3bda 100644
--- a/easybuild/toolchains/compiler/gcc.py
+++ b/easybuild/toolchains/compiler/gcc.py
@@ -78,6 +78,15 @@ class Gcc(Compiler):
COMPILER_UNIQUE_OPTION_MAP['strict'] = no_recip_alternative
COMPILER_UNIQUE_OPTION_MAP['precise'] = no_recip_alternative
+ # gcc on RISC-V does not support -mno-recip, -mieee-fp, -mfno-math-errno...
+ # https://gcc.gnu.org/onlinedocs/gcc/RISC-V-Options.html
+ # there are no good alternatives, so stick to the default flags
+ if systemtools.get_cpu_family() == systemtools.RISCV:
+ COMPILER_UNIQUE_OPTION_MAP['strict'] = []
+ COMPILER_UNIQUE_OPTION_MAP['precise'] = []
+ COMPILER_UNIQUE_OPTION_MAP['loose'] = ['fno-math-errno']
+ COMPILER_UNIQUE_OPTION_MAP['verloose'] = ['fno-math-errno']
+
# used when 'optarch' toolchain option is enabled (and --optarch is not specified)
COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {
(systemtools.AARCH32, systemtools.ARM): 'mcpu=native', # implies -march=native and -mtune=native
diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py
index 1a21b21943..c33b5ee1a2 100644
--- a/easybuild/toolchains/compiler/intel_compilers.py
+++ b/easybuild/toolchains/compiler/intel_compilers.py
@@ -48,7 +48,8 @@ class IntelCompilers(IntelIccIfort):
'oneapi': (None, "Use oneAPI compilers icx/icpx/ifx instead of classic compilers"),
'oneapi_c_cxx': (None, "Use oneAPI C/C++ compilers icx/icpx instead of classic Intel C/C++ compilers "
"(auto-enabled for Intel compilers version 2022.2.0, or newer)"),
- 'oneapi_fortran': (False, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler"),
+ 'oneapi_fortran': (None, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler "
+ "(auto-enabled for Intel compilers version 2024.0.0, or newer)"),
})
def _set_compiler_vars(self):
@@ -75,6 +76,9 @@ def set_variables(self):
# auto-enable use of oneAPI C/C++ compilers for sufficiently recent versions of Intel compilers
comp_ver = self.get_software_version(self.COMPILER_MODULE_NAME)[0]
if LooseVersion(comp_ver) >= LooseVersion('2022.2.0'):
+ if LooseVersion(comp_ver) >= LooseVersion('2024.0.0'):
+ if self.options.get('oneapi_fortran', None) is None:
+ self.options['oneapi_fortran'] = True
if self.options.get('oneapi_c_cxx', None) is None:
self.options['oneapi_c_cxx'] = True
diff --git a/easybuild/toolchains/gmpflf.py b/easybuild/toolchains/gmpflf.py
new file mode 100644
index 0000000000..30d10a90f7
--- /dev/null
+++ b/easybuild/toolchains/gmpflf.py
@@ -0,0 +1,45 @@
+##
+# Copyright 2013-2024 Ghent University
+#
+# This file is triple-licensed under GPLv2 (see below), MIT, and
+# BSD three-clause licenses.
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+EasyBuild support for gmpflf compiler toolchain (includes GCC, MPICH, FlexiBLAS, LAPACK, ScaLAPACK and FFTW).
+
+Authors:
+
+* Richard Topouchian (University of Bergen)
+"""
+from easybuild.toolchains.gmpich import Gmpich
+from easybuild.toolchains.gfbf import Gfbf
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+from easybuild.toolchains.linalg.scalapack import ScaLAPACK
+
+
+class Gmpflf(Gmpich, FlexiBLAS, ScaLAPACK, Fftw):
+ """Compiler toolchain with GCC, MPICH, FlexiBLAS, ScaLAPACK and FFTW."""
+ NAME = 'gmpflf'
+ SUBTOOLCHAIN = [Gmpich.NAME, Gfbf.NAME]
diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py
index 8a32d684d9..9dc89da827 100644
--- a/easybuild/toolchains/linalg/intelmkl.py
+++ b/easybuild/toolchains/linalg/intelmkl.py
@@ -142,6 +142,7 @@ def _set_blas_variables(self):
self.variables.nappend_el('CFLAGS', 'DMKL_ILP64')
# exact paths/linking statements depend on imkl version
+ root = self.get_software_root(self.BLAS_MODULE_NAME)[0]
found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0]
ver = LooseVersion(found_version)
if ver < LooseVersion('10.3'):
@@ -156,6 +157,8 @@ def _set_blas_variables(self):
found_version)
else:
if ver >= LooseVersion('2021'):
+ if os.path.islink(os.path.join(root, 'mkl', 'latest')):
+ found_version = os.readlink(os.path.join(root, 'mkl', 'latest'))
basedir = os.path.join('mkl', found_version)
else:
basedir = 'mkl'
diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py
index 487e6372a3..f5ce1ebc01 100644
--- a/easybuild/tools/build_details.py
+++ b/easybuild/tools/build_details.py
@@ -31,8 +31,8 @@
* Stijn De Weirdt (Ghent University)
"""
import time
+from collections import OrderedDict
from easybuild.tools.filetools import det_size
-from easybuild.tools.py2vs3 import OrderedDict
from easybuild.tools.systemtools import get_system_info
from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 6bec64764c..2dd71388cd 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -120,6 +120,8 @@
DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders'
DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild")
DEFAULT_REPOSITORY = 'FileRepository'
+EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io'
+DEFAULT_EXTRA_SOURCE_URLS = (EASYBUILD_SOURCES_URL,)
# Filter these CUDA libraries by default from the RPATH sanity check.
# These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build
# against the stub versions, but use the libraries that come with the CUDA driver at runtime. That means they should
@@ -259,6 +261,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'rpath_override_dirs',
'required_linked_shared_libs',
'skip',
+ 'software_commit',
'stop',
'subdir_user_modules',
'sysroot',
@@ -307,6 +310,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'set_gid_bit',
'silence_hook_trigger',
'skip_extensions',
+ 'skip_sanity_check',
'skip_test_cases',
'skip_test_step',
'sticky_bit',
@@ -391,6 +395,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'defaultopt': [
'default_opt_level',
],
+ DEFAULT_EXTRA_SOURCE_URLS: [
+ 'extra_source_urls',
+ ],
DEFAULT_ALLOW_LOADED_MODULES: [
'allow_loaded_modules',
],
diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py
index b4387454ba..d5b9cd4628 100644
--- a/easybuild/tools/docs.py
+++ b/easybuild/tools/docs.py
@@ -40,6 +40,7 @@
import inspect
import json
import os
+from collections import OrderedDict
from easybuild.tools import LooseVersion
from easybuild.base import fancylogger
@@ -60,7 +61,7 @@
from easybuild.tools.config import build_option
from easybuild.tools.filetools import read_file
from easybuild.tools.modules import modules_tool
-from easybuild.tools.py2vs3 import OrderedDict, ascii_lowercase
+from easybuild.tools.py2vs3 import ascii_lowercase
from easybuild.tools.toolchain.toolchain import DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
from easybuild.tools.toolchain.utilities import search_toolchain
from easybuild.tools.utilities import INDENT_2SPACES, INDENT_4SPACES
@@ -788,7 +789,7 @@ def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False
if isinstance(ec, dict):
template_values = template_constant_dict(ec)
for key in keys:
- if '%(' in info[key]:
+ if info[key] and '%(' in info[key]:
try:
info[key] = info[key] % template_values
except (KeyError, TypeError, ValueError) as err:
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 695043a595..06ca1dddb5 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -1242,12 +1242,16 @@ def calc_block_checksum(path, algorithm):
return algorithm.hexdigest()
-def verify_checksum(path, checksums):
+def verify_checksum(path, checksums, computed_checksums=None):
"""
Verify checksum of specified file.
:param path: path of file to verify checksum of
- :param checksums: checksum values (and type, optionally, default is MD5), e.g., 'af314', ('sha', '5ec1b')
+ :param checksums: checksum values to compare to
+ (and type, optionally, default is MD5), e.g., 'af314', ('sha', '5ec1b')
+ :param computed_checksums: Optional dictionary of (current) checksum(s) for this file
+ indexed by the checksum type (e.g. 'sha256').
+ Each existing entry will be used, missing ones will be computed.
"""
filename = os.path.basename(path)
@@ -1303,8 +1307,14 @@ def verify_checksum(path, checksums):
"2-tuple (type, value), or tuple of alternative checksum specs.",
checksum)
- actual_checksum = compute_checksum(path, typ)
- _log.debug("Computed %s checksum for %s: %s (correct checksum: %s)" % (typ, path, actual_checksum, checksum))
+ if computed_checksums is not None and typ in computed_checksums:
+ actual_checksum = computed_checksums[typ]
+ computed_str = 'Precomputed'
+ else:
+ actual_checksum = compute_checksum(path, typ)
+ computed_str = 'Computed'
+ _log.debug("%s %s checksum for %s: %s (correct checksum: %s)" %
+ (computed_str, typ, path, actual_checksum, checksum))
if actual_checksum != checksum:
return False
@@ -2409,12 +2419,16 @@ def copy_file(path, target_path, force_in_dry_run=False):
try:
# check whether path to copy exists (we could be copying a broken symlink, which is supported)
path_exists = os.path.exists(path)
+ # If target is a folder, the target_path will be a file with the same name inside the folder
+ if os.path.isdir(target_path):
+ target_path = os.path.join(target_path, os.path.basename(path))
target_exists = os.path.exists(target_path)
+
if target_exists and path_exists and os.path.samefile(path, target_path):
_log.debug("Not copying %s to %s since files are identical", path, target_path)
# if target file exists and is owned by someone else than the current user,
- # try using shutil.copyfile to just copy the file contents
- # since shutil.copy2 will fail when trying to copy over file metadata (since chown requires file ownership)
+ # copy just the file contents (shutil.copyfile instead of shutil.copy2)
+ # since copying the file metadata/permissions will fail since chown requires file ownership
elif target_exists and os.stat(target_path).st_uid != os.getuid():
shutil.copyfile(path, target_path)
_log.info("Copied contents of file %s to %s", path, target_path)
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 68ddba632a..f48433e23c 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -45,6 +45,7 @@
import sys
import tempfile
import pwd
+from collections import OrderedDict
import easybuild.tools.environment as env
from easybuild.base import fancylogger # build_log should always stay there, to ensure EasyBuildLog
@@ -69,9 +70,9 @@
from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL
-from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT
+from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS
from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
-from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS
+from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS
from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE
from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS
from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS
@@ -97,7 +98,7 @@
from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators
from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes
from easybuild.tools.modules import Lmod
-from easybuild.tools.py2vs3 import OrderedDict, string_type
+from easybuild.tools.py2vs3 import string_type
from easybuild.tools.robot import det_robot_path
from easybuild.tools.run import run_cmd
from easybuild.tools.package.utilities import avail_package_naming_schemes
@@ -407,6 +408,8 @@ def override_options(self):
None, 'store_true', False),
'extra-modules': ("List of extra modules to load after setting up the build environment",
'strlist', 'extend', None),
+ "extra-source-urls": ("Specify URLs to fetch sources from in addition to those in the easyconfig",
+ "urltuple", "add_flex", DEFAULT_EXTRA_SOURCE_URLS, {'metavar': 'URL[|URL]'}),
'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, "
"implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False),
'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, "
@@ -438,11 +441,11 @@ def override_options(self):
"(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None),
'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, "
"(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None),
- 'http-header-fields-urlpat': ("Set extra HTTP header FIELDs when downloading files from URL PATterns. "
- "To not log sensitive values, specify a file containing newline separated "
- "FIELDs. e.g. \"^https://www.example.com::/path/to/headers.txt\" or "
- "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\".",
- None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}),
+ 'http-header-fields-urlpat': ("Set extra HTTP header FIELD when downloading files from URL PATterns. "
+ "Use FILE (to hide sensitive values) and newline separated FIELDs in the "
+ "same format. e.g. \"^https://www.example.com::path/to/headers.txt\" or "
+ "\"client[A-z0-9]*.example.com:: Authorization: Basic token\".",
+ None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FIELDVALUE|FILE'}),
'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False),
'ignore-test-failure': ("Ignore a failing test step", None, 'store_true', False),
'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False),
@@ -509,8 +512,14 @@ def override_options(self):
'silence-hook-trigger': ("Suppress printing of debug message every time a hook is triggered",
None, 'store_true', False),
'skip-extensions': ("Skip installation of extensions", None, 'store_true', False),
+ 'skip-sanity-check': ("Skip running the sanity check step "
+ "(e.g. testing for installed files or running basic commands)",
+ None, 'store_true', False),
'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'),
'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False),
+ 'software-commit': (
+ "Git commit to use for the target software build (robot capabilities are automatically disabled)",
+ None, 'store', None),
'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False),
'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include",
None, 'store', None),
@@ -656,7 +665,8 @@ def informative_options(self):
'check-conflicts': ("Check for version conflicts in dependency graphs", None, 'store_true', False),
'check-eb-deps': ("Check presence and version of (required and optional) EasyBuild dependencies",
None, 'store_true', False),
- 'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.'}),
+ 'dep-graph': ("Create dependency graph. Output format depends on , e.g. 'dot', 'png', 'pdf', 'gv'.",
+ None, 'store', None, {'metavar': 'depgraph.'}),
'dump-env-script': ("Dump source script to set up build environment based on toolchain/dependencies",
None, 'store_true', False),
'last-log': ("Print location to EasyBuild log file of last (failed) session", None, 'store_true', False),
@@ -1098,6 +1108,12 @@ def _postprocess_checks(self):
else:
raise EasyBuildError("Specified sysroot '%s' does not exist!", self.options.sysroot)
+ # 'software_commit' is specific to a particular software package and cannot be used with 'robot'
+ if self.options.software_commit:
+ if self.options.robot is not None:
+ print_warning("To allow use of --software-commit robot resolution is being disabled")
+ self.options.robot = None
+
self.log.info("Checks on configuration options passed")
def get_cfg_opt_abs_path(self, opt_name, path):
@@ -1590,7 +1606,7 @@ def check_included_multiple(included_easyblocks_from, source):
check_included_multiple(included_from_commit, "commit %s" % easyblock_commit)
for easyblock in included_from_commit:
- print_msg("easyblock %s included from comit %s" % (easyblock, easyblock_commit), log=log)
+ print_msg("easyblock %s included from commit %s" % (easyblock, easyblock_commit), log=log)
include_easyblocks(options.tmpdir, easyblocks_from_commit)
diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py
index b6da8a9245..a8f98480f9 100644
--- a/easybuild/tools/output.py
+++ b/easybuild/tools/output.py
@@ -32,10 +32,10 @@
* Jørgen Nordmoen (University of Oslo)
"""
import functools
+from collections import OrderedDict
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style
-from easybuild.tools.py2vs3 import OrderedDict
try:
from rich.console import Console, Group
diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py
index f619279060..b52f55a1a2 100644
--- a/easybuild/tools/py2vs3/py2.py
+++ b/easybuild/tools/py2vs3/py2.py
@@ -52,7 +52,7 @@
# reload function (built-in in Python 2)
-reload = reload
+reload = reload # noqa: F821
# string type that can be used in 'isinstance' calls
string_type = basestring
@@ -89,9 +89,12 @@ def subprocess_terminate(proc, timeout):
proc.terminate()
+# Wrapped in exec to avoid invalid syntax warnings for Python 3
+exec('''
def raise_with_traceback(exception_class, message, traceback):
"""Raise exception of specified class with given message and traceback."""
raise exception_class, message, traceback # noqa: E999
+''')
def extract_method_name(method_func):
diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index 73eb657510..24e910fb94 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -42,6 +42,7 @@
import sys
import termios
import warnings
+from collections import OrderedDict
from ctypes.util import find_library
from socket import gethostname
from easybuild.tools.py2vs3 import subprocess_popen_text
@@ -64,7 +65,7 @@
from easybuild.tools.build_log import EasyBuildError, print_warning
from easybuild.tools.config import IGNORE
from easybuild.tools.filetools import is_readable, read_file, which
-from easybuild.tools.py2vs3 import OrderedDict, string_type
+from easybuild.tools.py2vs3 import string_type
from easybuild.tools.run import run_cmd
@@ -735,7 +736,6 @@ def get_os_name():
# platform.linux_distribution was removed in Python 3.8,
# see https://docs.python.org/2/library/platform.html#platform.linux_distribution
if hasattr(platform, 'linux_distribution'):
- # platform.linux_distribution is more useful, but only available since Python 2.6
# this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=PendingDeprecationWarning)
diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py
index bbe7f386ad..fb2601805d 100644
--- a/easybuild/tools/version.py
+++ b/easybuild/tools/version.py
@@ -45,7 +45,7 @@
# recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
# UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
# This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.9.2')
+VERSION = LooseVersion('4.9.3')
UNKNOWN = 'UNKNOWN'
UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'
diff --git a/eb b/eb
index ad14ba61d0..0a510c70ad 100755
--- a/eb
+++ b/eb
@@ -40,8 +40,8 @@ keyboard_interrupt() {
trap keyboard_interrupt SIGINT
-# Python 2.6+ or 3.5+ required
-REQ_MIN_PY2VER=6
+# Python 2.7+ or 3.5+ required
+REQ_MIN_PY2VER=7
REQ_MIN_PY3VER=5
EASYBUILD_MAIN='easybuild.main'
diff --git a/requirements.txt b/requirements.txt
index 85a9df78e7..d2ce2ccdc1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,12 @@
# keyring is required to provide GitHub token to EasyBuild;
# for recent versions of keyring, keyrings.alt must be installed too
-keyring
+# 19.0 dropped Python 2 support
+keyring<19.0; python_version < '3.0'
+keyring; python_version >= '3.0'
keyrings.alt
# GitPython 3.1.15 deprecates Python 3.5
-GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6'
+GitPython<3.1.15; python_version >= '3.0' and python_version < '3.6'
GitPython; python_version >= '3.6' or python_version <= '3.0'
# autopep8
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
index 4b63dc605b..c43c84be0e 100644
--- a/test/framework/easyconfig.py
+++ b/test/framework/easyconfig.py
@@ -38,6 +38,7 @@
import sys
import tempfile
import textwrap
+from collections import OrderedDict
from easybuild.tools import LooseVersion
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
@@ -72,7 +73,7 @@
from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
from easybuild.tools.options import parse_external_modules_metadata
-from easybuild.tools.py2vs3 import OrderedDict, reload
+from easybuild.tools.py2vs3 import reload
from easybuild.tools.robot import det_robot_path, resolve_dependencies
from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64
from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext, get_os_name, get_os_version
@@ -138,10 +139,13 @@ def tearDown(self):
def test_empty(self):
""" empty files should not parse! """
+ self.assertErrorRegex(EasyBuildError, "expected a valid path", EasyConfig, "")
self.contents = "# empty string"
self.prep()
self.assertRaises(EasyBuildError, EasyConfig, self.eb_file)
- self.assertErrorRegex(EasyBuildError, "expected a valid path", EasyConfig, "")
+ self.contents = ""
+ self.prep()
+ self.assertErrorRegex(EasyBuildError, "is empty", EasyConfig, self.eb_file)
def test_mandatory(self):
""" make sure all checking of mandatory parameters works """
@@ -1447,6 +1451,35 @@ def test_sysroot_template(self):
self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix)
self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix)
+ def test_software_commit_template(self):
+ """Test the %(software_commit)s template"""
+
+ test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += '\nconfigopts = "--some-opt=%(software_commit)s"'
+ test_ec_txt += '\nbuildopts = "--some-opt=%(software_commit)s"'
+ test_ec_txt += '\ninstallopts = "--some-opt=%(software_commit)s"'
+ write_file(test_ec, test_ec_txt)
+
+ # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None)
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['configopts'], "--some-opt=")
+ self.assertEqual(ec['buildopts'], "--some-opt=")
+ self.assertEqual(ec['installopts'], "--some-opt=")
+
+ # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None)
+ # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist
+ software_commit = '1234bc'
+ update_build_option('software_commit', software_commit)
+
+ ec = EasyConfig(test_ec)
+ self.assertEqual(ec['configopts'], "--some-opt=%s" % software_commit)
+ self.assertEqual(ec['buildopts'], "--some-opt=%s" % software_commit)
+ self.assertEqual(ec['installopts'], "--some-opt=%s" % software_commit)
+
def test_constant_doc(self):
"""test constant documentation"""
doc = avail_easyconfig_constants()
@@ -1634,6 +1667,10 @@ def test_get_easyblock_class(self):
self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY')
self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None)
+ # Test passing neither easyblock nor name
+ self.assertErrorRegex(EasyBuildError, "neither name nor easyblock were specified", get_easyblock_class, None)
+ self.assertEqual(get_easyblock_class(None, error_on_missing_easyblock=False), None)
+
# also test deprecated default_fallback named argument
self.assertErrorRegex(EasyBuildError, "DEPRECATED", get_easyblock_class, None, name='gzip',
default_fallback=False)
@@ -3337,6 +3374,7 @@ def test_template_constant_dict(self):
'nameletter': 'g',
'nameletterlower': 'g',
'parallel': None,
+ 'software_commit': '',
'sysroot': '',
'toolchain_name': 'foss',
'toolchain_version': '2018a',
@@ -3419,6 +3457,7 @@ def test_template_constant_dict(self):
'pyminver': '7',
'pyshortver': '3.7',
'pyver': '3.7.2',
+ 'software_commit': '',
'sysroot': '',
'version': '0.01',
'version_major': '0',
@@ -3484,6 +3523,7 @@ def test_template_constant_dict(self):
'namelower': 'foo',
'nameletter': 'f',
'nameletterlower': 'f',
+ 'software_commit': '',
'sysroot': '',
'version': '1.2.3',
'version_major': '1',
@@ -4606,7 +4646,7 @@ def test_cuda_compute_capabilities(self):
toolchain = SYSTEM
cuda_compute_capabilities = ['5.1', '7.0', '7.1']
installopts = '%(cuda_compute_capabilities)s'
- preinstallopts = '%(cuda_cc_space_sep)s'
+ preinstallopts = 'period="%(cuda_cc_space_sep)s" noperiod="%(cuda_cc_space_sep_no_period)s"'
prebuildopts = '%(cuda_cc_semicolon_sep)s'
configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"'
preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"'
@@ -4615,7 +4655,7 @@ def test_cuda_compute_capabilities(self):
ec = EasyConfig(self.eb_file)
self.assertEqual(ec['installopts'], '5.1,7.0,7.1')
- self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1')
+ self.assertEqual(ec['preinstallopts'], 'period="5.1 7.0 7.1" noperiod="51 70 71"')
self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1')
self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" '
'space="sm_51 sm_70 sm_71"')
@@ -4625,7 +4665,7 @@ def test_cuda_compute_capabilities(self):
init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']})
ec = EasyConfig(self.eb_file)
self.assertEqual(ec['installopts'], '4.2,6.3')
- self.assertEqual(ec['preinstallopts'], '4.2 6.3')
+ self.assertEqual(ec['preinstallopts'], 'period="4.2 6.3" noperiod="42 63"')
self.assertEqual(ec['prebuildopts'], '4.2;6.3')
self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" '
'space="sm_42 sm_63"')
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 63cd0a93b7..1a0294a02f 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -33,6 +33,7 @@
"""
import datetime
import glob
+import logging
import os
import re
import shutil
@@ -297,10 +298,26 @@ def test_checksums(self):
'b7297da8b547d5e74b851d7c4e475900cec4744df0f887ae5c05bf1757c224b4',
}
+ old_log_level = ft._log.getEffectiveLevel()
+ ft._log.setLevel(logging.DEBUG)
# make sure checksums computation/verification is correct
for checksum_type, checksum in known_checksums.items():
self.assertEqual(ft.compute_checksum(fp, checksum_type=checksum_type), checksum)
- self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum)))
+ with self.log_to_testlogfile():
+ self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum)))
+ self.assertIn('Computed ' + checksum_type, ft.read_file(self.logfile))
+ # Passing precomputed checksums reuses it
+ with self.log_to_testlogfile():
+ computed_checksums = {checksum_type: checksum}
+ self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum), computed_checksums))
+ self.assertIn('Precomputed ' + checksum_type, ft.read_file(self.logfile))
+ # If the type isn't contained the checksum will be computed
+ with self.log_to_testlogfile():
+ computed_checksums = {'doesnt exist': 'checksum'}
+ self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum), computed_checksums))
+ self.assertIn('Computed ' + checksum_type, ft.read_file(self.logfile))
+
+ ft._log.setLevel(old_log_level)
# default checksum type is MD5
self.assertEqual(ft.compute_checksum(fp), known_checksums['md5'])
@@ -1839,6 +1856,19 @@ def test_copy_file(self):
# printing this message will make test suite fail in Travis/GitHub CI,
# since we check for unexpected output produced by the tests
print("Skipping overwrite-file-owned-by-other-user copy_file test (%s is missing)" % test_file_to_overwrite)
+ # Copy a file to a directory owned by some other user, e.g. /tmp (owned by root)
+ # This might be a common choice for e.g. --copy-ec
+ target_file_path = tempfile.mktemp("easybuild", dir="/tmp")
+ test_file_to_copy = os.path.join(self.test_prefix, os.path.basename(target_file_path))
+ ft.write_file(test_file_to_copy, test_file_contents)
+ try:
+ ft.copy_file(test_file_to_copy, '/tmp')
+ self.assertEqual(ft.read_file(target_file_path), test_file_contents)
+ finally:
+ try:
+ os.remove(target_file_path)
+ except FileNotFoundError:
+ pass
# also test behaviour of copy_file under --dry-run
build_options = {
diff --git a/test/framework/github.py b/test/framework/github.py
index c4eb8c8854..d899ad06a1 100644
--- a/test/framework/github.py
+++ b/test/framework/github.py
@@ -816,18 +816,21 @@ def test_github_det_commit_status(self):
res = gh.det_commit_status('easybuilders', GITHUB_REPO, commit_sha, GITHUB_TEST_ACCOUNT)
self.assertEqual(res, None)
- # recent commit (2023-04-11) with cancelled checks (GitHub Actions only)
- commit_sha = 'c074f0bb3110c27d9969c3d0b19dde3eca868bd4'
+ # recent commit with cancelled checks (GitHub Actions only);
+ # to update, use https://github.com/easybuilders/easybuild-easyconfigs/actions?query=is%3Acancelled
+ commit_sha = '52b964c3387d6d6f149ec304f9e23f535e799957'
res = gh.det_commit_status('easybuilders', 'easybuild-easyconfigs', commit_sha, GITHUB_TEST_ACCOUNT)
self.assertEqual(res, 'cancelled')
- # recent commit (2023-04-10) with failing checks (GitHub Actions only)
- commit_sha = '1b4a45c62d7deaf19125756c46dc8f011fef66e1'
+ # recent commit with failing checks (GitHub Actions only)
+ # to update, use https://github.com/easybuilders/easybuild-easyconfigs/actions?query=is%3Afailure
+ commit_sha = '85e6c2bbc2fd515a1d4dab607b8d43d0a1ed668f'
res = gh.det_commit_status('easybuilders', 'easybuild-easyconfigs', commit_sha, GITHUB_TEST_ACCOUNT)
self.assertEqual(res, 'failure')
- # recent commit (2023-04-10) with successful checks (GitHub Actions only)
- commit_sha = '56812a347acbaaa87f229fe319425020fe399647'
+ # recent commit with successful checks (GitHub Actions only)
+ # to update, use https://github.com/easybuilders/easybuild-easyconfigs/actions?query=is%3Asuccess
+ commit_sha = 'f82a563b8e1f8118c7c3ab23374d0e28e1691fea'
res = gh.det_commit_status('easybuilders', 'easybuild-easyconfigs', commit_sha, GITHUB_TEST_ACCOUNT)
self.assertEqual(res, 'success')
diff --git a/test/framework/modules.py b/test/framework/modules.py
index 195bf0339c..8ba4f4ac65 100644
--- a/test/framework/modules.py
+++ b/test/framework/modules.py
@@ -55,7 +55,7 @@
# number of modules included for testing purposes
-TEST_MODULES_COUNT = 110
+TEST_MODULES_COUNT = 111
class ModulesTest(EnhancedTestCase):
diff --git a/test/framework/modules/intel-compilers/2024.0.0 b/test/framework/modules/intel-compilers/2024.0.0
new file mode 100644
index 0000000000..a5c0267f9d
--- /dev/null
+++ b/test/framework/modules/intel-compilers/2024.0.0
@@ -0,0 +1,37 @@
+#%Module
+proc ModulesHelp { } {
+ puts stderr {
+
+Description
+===========
+Intel C, C++ & Fortran compilers (classic and oneAPI)
+
+
+More information
+================
+ - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html
+ }
+}
+
+module-whatis {Description: Intel C, C++ & Fortran compilers (classic and oneAPI)}
+module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html}
+module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html}
+
+set root /tmp/intel-compilers/2024.0.0
+
+conflict intel-compilers
+
+prepend-path CPATH $root/tbb/2021.11/include
+prepend-path LD_LIBRARY_PATH $root/compiler/2024.0/linux/lib
+prepend-path LD_LIBRARY_PATH $root/tbb/2021.11/lib/intel64/gcc4.8
+prepend-path LIBRARY_PATH $root/compiler/2024.0/linux/lib
+prepend-path LIBRARY_PATH $root/tbb/2021.11/lib/intel64/gcc4.8
+prepend-path MANPATH $root/compiler/2024.0/share/man
+prepend-path OCL_ICD_FILENAMES $root/compiler/2024.0/lib/libintelocl.so
+prepend-path PATH $root/compiler/2024.0/bin
+prepend-path TBBROOT $root/tbb/2021.11
+setenv EBROOTINTELMINCOMPILERS "$root"
+setenv EBVERSIONINTELMINCOMPILERS "2024.0.0"
+setenv EBDEVELINTELMINCOMPILERS "$root/easybuild/Core-intel-compilers-2024.0.0-easybuild-devel"
+
+# Built with EasyBuild version 4.8.2
diff --git a/test/framework/options.py b/test/framework/options.py
index 94c4372f91..2208465db0 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -443,6 +443,27 @@ def test_ignore_test_failure(self):
error_pattern = 'Found both ignore-test-failure and skip-test-step enabled'
self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+ def test_skip_sanity_check(self):
+ """Test skipping of sanity check step (--skip-sanity-check)."""
+
+ topdir = os.path.abspath(os.path.dirname(__file__))
+ toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, read_file(toy_ec) + "\nsanity_check_commands = ['this_will_fail']")
+
+ args = [test_ec, '--rebuild']
+ err_msg = "Sanity check failed"
+ self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True)
+
+ args.append('--skip-sanity-check')
+ outtext = self.eb_main(args, do_build=True, raise_error=True)
+ self.assertNotIn('sanity checking...', outtext)
+
+ # Passing skip and only options is disallowed
+ args.append('--sanity-check-only')
+ error_pattern = 'Found both skip-sanity-check and sanity-check-only enabled'
+ self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+
def test_job(self):
"""Test submitting build as a job."""
@@ -591,6 +612,8 @@ def run_test(fmt=None):
r'^``%\(arch\)s``\s+System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)\s*$',
r'^``%\(cuda_cc_space_sep\)s``\s+Space-separated list of CUDA compute capabilities\s*$',
r'^``SOURCE_TAR_GZ``\s+Source \.tar\.gz bundle\s+``%\(name\)s-%\(version\)s.tar.gz``\s*$',
+ r'^``%\(software_commit\)s``\s+Git commit id to use for the software as specified '
+ 'by --software-commit command line option',
]
else:
pattern_lines = [
@@ -603,6 +626,8 @@ def run_test(fmt=None):
r'^\s+%\(arch\)s: System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)$',
r'^\s+%\(cuda_cc_space_sep\)s: Space-separated list of CUDA compute capabilities$',
r'^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)$',
+ r'^\s+%\(software_commit\)s: Git commit id to use for the software as specified '
+ 'by --software-commit command line option',
]
for pattern_line in pattern_lines:
@@ -2225,7 +2250,7 @@ def test_xxx_include_easyblocks_from_commit(self):
import easybuild.easyblocks.generic
reload(easybuild.easyblocks.generic)
- pattern = "== easyblock binary.py included from comit %s" % test_commit
+ pattern = "== easyblock binary.py included from commit %s" % test_commit
self.assertEqual(stderr, '')
self.assertIn(pattern, stdout)
@@ -6045,7 +6070,7 @@ def test_inject_checksums(self):
patterns = [
r"^== injecting sha256 checksums in .*/test\.eb$",
r"^== fetching sources & patches for test\.eb\.\.\.$",
- r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+\.\.\.$",
+ r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+$",
r"^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$",
r"^== \* toy-0.0\.tar\.gz: %s$" % toy_source_sha256,
r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: %s$" % toy_patch_sha256,
@@ -6173,7 +6198,7 @@ def test_inject_checksums(self):
patterns = [
r"^== injecting md5 checksums in .*/test\.eb$",
r"^== fetching sources & patches for test\.eb\.\.\.$",
- r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+\.\.\.$",
+ r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+$",
r"^== injecting md5 checksums for sources & patches in test\.eb\.\.\.$",
r"^== \* toy-0.0\.tar\.gz: be662daa971a640e40be5c804d9d7d10$",
r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: a99f2a72cee1689a2f7e3ace0356efb1$",
@@ -6813,6 +6838,23 @@ def test_sysroot(self):
os.environ['EASYBUILD_SYSROOT'] = doesnotexist
self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True)
+ def test_software_commit(self):
+ """Test use of --software-commit option."""
+
+ software_commit = "23be34"
+ software_commit_arg = '--software-commit=' + software_commit
+ # Add robot to also test that it gets disabled
+ stdout, stderr = self._run_mock_eb([software_commit_arg, '--show-config', '--robot'], raise_error=True)
+
+ warning_regex = re.compile(r'.*WARNING:.*--software-commit robot resolution is being disabled.*', re.M)
+ software_commit_regex = re.compile(r'^software-commit\s*\(C\) = %s$' % software_commit, re.M)
+ robot_regex = re.compile(r'^robot\s*\(C\) = .*', re.M)
+
+ self.assertTrue(warning_regex.search(stderr), "Pattern '%s' not found in: %s" % (warning_regex, stderr))
+ self.assertTrue(software_commit_regex.search(stdout),
+ "Pattern '%s' not found in: %s" % (software_commit_regex, stdout))
+ self.assertFalse(robot_regex.search(stdout), "Pattern '%s' found in: %s" % (robot_regex, stdout))
+
def test_accept_eula_for(self):
"""Test --accept-eula-for configuration option."""
diff --git a/test/framework/repository.py b/test/framework/repository.py
index 720e0fa8e8..feb96a13cf 100644
--- a/test/framework/repository.py
+++ b/test/framework/repository.py
@@ -41,7 +41,6 @@
from easybuild.tools.filetools import read_file
from easybuild.tools.repository.filerepo import FileRepository
from easybuild.tools.repository.gitrepo import GitRepository
-from easybuild.tools.repository.hgrepo import HgRepository
from easybuild.tools.repository.svnrepo import SvnRepository
from easybuild.tools.repository.repository import init_repository
from easybuild.tools.run import run_cmd
@@ -129,27 +128,6 @@ def test_svnrepo(self):
self.assertExists(os.path.join(repo.wc, 'trunk', 'README.md'))
shutil.rmtree(repo.wc)
- # this test is disabled because it fails in Travis as a result of bitbucket disabling TLS 1.0/1.1
- # we can consider re-enabling it when moving to a more recent Ubuntu version in the Travis config
- # (which implies dropping support for Python 2.6)
- # cfr. https://github.com/easybuilders/easybuild-framework/pull/2678
- def DISABLED_test_hgrepo(self):
- """Test using HgRepository."""
- # only run this test if pysvn Python module is available
- try:
- import hglib # noqa
- except ImportError:
- print("(skipping HgRepository test)")
- return
-
- # GitHub also supports SVN
- test_repo_url = 'https://kehoste@bitbucket.org/kehoste/testrepository'
-
- repo = HgRepository(test_repo_url)
- repo.init()
- self.assertExists(os.path.join(repo.wc, 'README'))
- shutil.rmtree(repo.wc)
-
def test_init_repository(self):
"""Test use of init_repository function."""
repo = init_repository('FileRepository', self.path)
diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index 0e53639491..beeb2e3512 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -1201,7 +1201,7 @@ def test_fft_env_vars_intel(self):
self.assertEqual(tc.get_variable('LIBFFT'), libfft)
self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt)
- fft_lib_dir = os.path.join(modules.get_software_root('imkl'), 'mkl/2021.4.0/lib/intel64')
+ fft_lib_dir = os.path.join(modules.get_software_root('imkl'), 'mkl/2021.4/lib/intel64')
self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir)
tc = self.get_toolchain('intel', version='2021b')
@@ -1321,8 +1321,11 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'):
])
write_file(imkl_fftw_module_path, imkl_fftw_mod_txt)
- subdir = 'mkl/%s/lib/intel64' % imklver
+ # put "latest" symbolic link to short version, used in newer MKL
+ imklshortver = '.'.join(imklver.split('.')[:2])
+ subdir = 'mkl/%s/lib/intel64' % imklshortver
os.makedirs(os.path.join(imkl_dir, subdir))
+ os.symlink(imklshortver, os.path.join(imkl_dir, 'mkl', 'latest'))
for fftlib in mkl_libs:
write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo')
subdir = 'lib'
@@ -1496,6 +1499,37 @@ def test_intel_toolchain_oneapi(self):
self.assertEqual(os.getenv('F90'), 'ifx')
self.assertEqual(os.getenv('FC'), 'ifx')
+ self.modtool.purge()
+ tc = self.get_toolchain('intel-compilers', version='2024.0.0')
+ tc.prepare()
+
+ # by default (for version >= 2024.0.0): oneAPI C/C++ compiler + oneAPI Fortran compiler
+ self.assertEqual(os.getenv('CC'), 'icx')
+ self.assertEqual(os.getenv('CXX'), 'icpx')
+ self.assertEqual(os.getenv('F77'), 'ifx')
+ self.assertEqual(os.getenv('F90'), 'ifx')
+ self.assertEqual(os.getenv('FC'), 'ifx')
+
+ self.modtool.purge()
+ tc = self.get_toolchain('intel-compilers', version='2024.0.0')
+ tc.set_options({'oneapi_fortran': False})
+ tc.prepare()
+ self.assertEqual(os.getenv('CC'), 'icx')
+ self.assertEqual(os.getenv('CXX'), 'icpx')
+ self.assertEqual(os.getenv('F77'), 'ifort')
+ self.assertEqual(os.getenv('F90'), 'ifort')
+ self.assertEqual(os.getenv('FC'), 'ifort')
+
+ self.modtool.purge()
+ tc = self.get_toolchain('intel-compilers', version='2024.0.0')
+ tc.set_options({'oneapi_c_cxx': False, 'oneapi_fortran': False})
+ tc.prepare()
+ self.assertEqual(os.getenv('CC'), 'icc')
+ self.assertEqual(os.getenv('CXX'), 'icpc')
+ self.assertEqual(os.getenv('F77'), 'ifort')
+ self.assertEqual(os.getenv('F90'), 'ifort')
+ self.assertEqual(os.getenv('FC'), 'ifort')
+
self.modtool.purge()
tc = self.get_toolchain('intel', version='2021b')
tc.set_options({'oneapi_c_cxx': True})
@@ -2203,7 +2237,8 @@ def test_compiler_cache(self):
"#!/bin/bash",
"echo 'This is a %s wrapper'" % cache_tool,
"NAME=${0##*/}",
- "comm=$(which -a $NAME | sed 1d)",
+ "comms=($(which -a $NAME))",
+ "comm=${comms[1]}", # First entry is this wrapper, take 2nd
"$comm $@",
"exit 0"
]