diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7327d876ba..e74b65cc79 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -18,6 +18,7 @@ jobs: fail-fast: false container: image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 + env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions steps: - name: Check out the repo uses: actions/checkout@v3 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index c9f42891ec..8a8b13af96 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -101,12 +101,12 @@ jobs: # and are only run after the PR gets merged GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} run: | - # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit; + # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit # tests that require a GitHub token are skipped automatically when no GitHub token is available if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then if [ ! -z $GITHUB_TOKEN ]; then - SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; - python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; + SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())" + python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')" fi echo "GitHub token installed!" else @@ -191,7 +191,17 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" + IGNORE_PATTERNS="no GitHub token available" + IGNORE_PATTERNS+="|skipping SvnRepository test" + IGNORE_PATTERNS+="|requires Lmod as modules tool" + IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device" + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]" + IGNORE_PATTERNS+="|from cryptography.* import " + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2" + IGNORE_PATTERNS+="|Blowfish" + IGNORE_PATTERNS+="|GC3Pie not available, skipping test" + IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved" + IGNORE_PATTERNS+="|algorithms.TripleDES" # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) diff --git a/.github/workflows/unit_tests_python2.yml b/.github/workflows/unit_tests_python2.yml index 1b921ee83c..aa00f9a2fd 100644 --- a/.github/workflows/unit_tests_python2.yml +++ b/.github/workflows/unit_tests_python2.yml @@ -16,6 +16,7 @@ jobs: # CentOS 7.9 container that already includes Lmod & co, # see https://github.com/easybuilders/easybuild-containers image: ghcr.io/easybuilders/centos-7.9-amd64 + env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions steps: - uses: actions/checkout@v3 diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 5a45be8a56..7f7ade0c1e 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,43 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.9.3 (14 September 2024) +-------------------------- + +update/bugfix release + +- various enhancements, including: + - add support for `--extra-source-urls` to fetch sources from additional URLs (#4079) + - add definition for gmpflf toolchain (#4566, #4571) + - reuse pre-computed checksums (#4569) + - add `cuda_cc_space_sep` variant that does not have periods (#4583) + - add `--skip-sanity-check` option (#4590) + - add `GNU_FTP_SOURCE` template constant (#4597) + - improve error messages for empty easyconfigs (#4603) + - improve help string for `--dep-graph` (#4610) + - only call `_sanity_check_step_extensions` if `--skip-extensions` is not set (#4620) + - add support for `--software-commit` and an associated template `%(software_commit)s` (#4628) +- various bug fixes, including: + - correctly evaluate result for `--dep-graph` (#4554) + - fix fetch progress bar showing to many files (#4568) + - resolve internal for imkl>=2021 version subdir via "latest" symlink (#4570) + - fix typo in message about including an easyblock from a commit (#4575) + - don't use special flags for `strict`, `precise`, `loose`, `veryloose` toolchain options on RISC-V (#4576) + - fix help text for `cuda_compute_capabilities` template (#4589) + - fix help message for `--http-headers-fields-urlpat` configuration option (#4594) + - fix `test_compiler_cache` in case `gcc` is available multiple times (#4599) + - handle post-install patches in check_checksums_for (#4605) + - fix `copy_file` with a folder as the target (#4609) + - allow for case where `homepage = None` when generating the docs (#4626) + - fix test_github_det_commit_status by using more recent commits (#4636) +- other changes: + - clean up code that was only there to support Python 2.6 + avoid syntax warnings when parsing py2vs3/py.p2 with Python 3.x (#3788) + - use Intel's oneAPI Fortran compiler by default for version 2024.0.0 and newer (`oneapi_fortran` toolchain option set to `True`) (#4567) + - allow using Node 16 actions in CI (#4574) + - remove a superflous check in `EasyBlock.run_all_steps` (#4623) + - remove trailing dots from backup message produced by --inject-checksums (#4632) + + v4.9.2 (12 June 2024) --------------------- diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 90248fb0cd..c3660f7cbf 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -97,11 +97,16 @@ def what_str_list_tuple(name): """Given name, return separator, class and helptext wrt separator. (Currently supports strlist, strtuple, pathlist, pathtuple) """ - sep = ',' - helpsep = 'comma' if name.startswith('path'): sep = os.pathsep helpsep = 'pathsep' + elif name.startswith('url'): + # | is one of the only characters not in the grammar for URIs (RFC3986) + sep = '|' + helpsep = '|' + else: + sep = ',' + helpsep = 'comma' klass = None if name.endswith('list'): @@ -182,6 +187,7 @@ class ExtOption(CompleterOption): - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings - pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings - the path separator is OS-dependent + - urllist, urltuple: convert string seperated by '|' to a list resp. tuple of strings """ EXTEND_SEPARATOR = ',' @@ -198,7 +204,7 @@ class ExtOption(CompleterOption): TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS - TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']] + TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path', 'url']] TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST} TYPE_CHECKER.update(Option.TYPE_CHECKER) TYPES = tuple(TYPE_STRLIST + list(Option.TYPES)) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 80cddf8819..0b76f5bff2 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -72,6 +72,7 @@ from easybuild.tools.build_log import print_error, print_msg, print_warning from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES +from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env @@ -105,9 +106,6 @@ from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION - -EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' - DEFAULT_BIN_LIB_SUBDIRS = ('bin', 'lib', 'lib64') MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP] @@ -674,14 +672,16 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): src_fn = os.path.basename(src_path) # report both MD5 and SHA256 checksums, since both are valid default checksum types + src_checksums = {} for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256): src_checksum = compute_checksum(src_path, checksum_type=checksum_type) + src_checksums[checksum_type] = src_checksum self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum) # verify checksum (if provided) self.log.debug('Verifying checksums for extension source...') fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0) - if verify_checksum(src_path, fn_checksum): + if verify_checksum(src_path, fn_checksum, src_checksums): self.log.info('Checksum for extension source %s verified', src_fn) elif build_option('ignore_checksums'): print_warning("Ignoring failing checksum verification for %s" % src_fn) @@ -700,12 +700,15 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): ext_src.update({'patches': ext_patches}) if verify_checksums: + computed_checksums = {} for patch in ext_patches: patch = patch['path'] + computed_checksums[patch] = {} # report both MD5 and SHA256 checksums, # since both are valid default checksum types for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256): checksum = compute_checksum(patch, checksum_type=checksum_type) + computed_checksums[patch][checksum_type] = checksum self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum) # verify checksum (if provided) @@ -715,7 +718,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): patch_fn = os.path.basename(patch) checksum = self.get_checksum_for(checksums, filename=patch_fn, index=idx+1) - if verify_checksum(patch, checksum): + if verify_checksum(patch, checksum, computed_checksums[patch]): self.log.info('Checksum for extension patch %s verified', patch_fn) elif build_option('ignore_checksums'): print_warning("Ignoring failing checksum verification for %s" % patch_fn) @@ -754,7 +757,9 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No """ srcpaths = source_paths() - update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename) + # We don't account for the checksums file in the progress bar + if filename != 'checksum.json': + update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename) if alt_location is None: location = self.name @@ -890,8 +895,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No source_urls = [] source_urls.extend(self.cfg['source_urls']) - # add https://sources.easybuild.io as fallback source URL - source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location)) + # Add additional URLs as configured. + for url in build_option("extra_source_urls"): + url += "/" + name_letter + "/" + location + source_urls.append(url) mkdir(targetdir, parents=True) @@ -2475,7 +2482,7 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): checksum_issues = [] sources = ent.get('sources', []) - patches = ent.get('patches', []) + patches = ent.get('patches', []) + ent.get('postinstallpatches', []) checksums = ent.get('checksums', []) # Single source should be re-wrapped as a list, and checksums with it if isinstance(sources, dict): @@ -3482,10 +3489,7 @@ def _sanity_check_step_extensions(self): """Sanity check on extensions (if any).""" failed_exts = [] - if build_option('skip_extensions'): - self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...") - return - elif not self.ext_instances: + if not self.ext_instances: # class instances for extensions may not be initialized yet here, # for example when using --module-only or --sanity-check-only self.prepare_for_extensions() @@ -3637,7 +3641,10 @@ def xs2str(xs): # also run sanity check for extensions (unless we are an extension ourselves) if not extension: - self._sanity_check_step_extensions() + if build_option('skip_extensions'): + self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...") + else: + self._sanity_check_step_extensions() linked_shared_lib_fails = self.sanity_check_linked_shared_libs() if linked_shared_lib_fails: @@ -3894,12 +3901,13 @@ def update_config_template_run_step(self): self.cfg.generate_template_values() def skip_step(self, step, skippable): - """Dedice whether or not to skip the specified step.""" + """Decide whether or not to skip the specified step.""" skip = False force = build_option('force') module_only = build_option('module_only') or self.cfg['module_only'] sanity_check_only = build_option('sanity_check_only') skip_extensions = build_option('skip_extensions') + skip_sanity_check = build_option('skip_sanity_check') skip_test_step = build_option('skip_test_step') skipsteps = self.cfg['skipsteps'] @@ -3927,6 +3935,10 @@ def skip_step(self, step, skippable): self.log.info("Skipping %s step because of sanity-check-only mode", step) skip = True + elif skip_sanity_check and step == SANITYCHECK_STEP: + self.log.info("Skipping %s step as request via skip-sanity-check", step) + skip = True + elif skip_extensions and step == EXTENSIONS_STEP: self.log.info("Skipping %s step as requested via skip-extensions", step) skip = True @@ -3937,9 +3949,9 @@ def skip_step(self, step, skippable): else: msg = "Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s, " - msg += "sanity_check_only: %s, skip_extensions: %s, skip_test_step: %s)" + msg += "sanity_check_only: %s, skip_extensions: %s, skip_test_step: %s, skip_sanity_check: %s)" self.log.debug(msg, step, skippable, self.skip, skipsteps, module_only, force, - sanity_check_only, skip_extensions, skip_test_step) + sanity_check_only, skip_extensions, skip_test_step, skip_sanity_check) return skip @@ -4089,7 +4101,7 @@ def run_all_steps(self, run_test_cases): Build and install this software. run_test_cases (bool): run tests after building (e.g.: make test) """ - if self.cfg['stop'] and self.cfg['stop'] == 'cfg': + if self.cfg['stop'] == 'cfg': return True steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt()) @@ -4708,7 +4720,7 @@ def make_checksum_lines(checksums, indent_level): # back up easyconfig file before injecting checksums ec_backup = back_up_file(ec['spec']) - print_msg("backup of easyconfig file saved to %s..." % ec_backup, log=_log) + print_msg("backup of easyconfig file saved to %s" % ec_backup, log=_log) # compute & inject checksums for sources/patches print_msg("injecting %s checksums for sources & patches in %s..." % (checksum_type, ec_fn), log=_log) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2b6aaa751a..a8c9a488b9 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -46,6 +46,7 @@ import functools import os import re +from collections import OrderedDict from contextlib import contextmanager import easybuild.tools.filetools as filetools @@ -74,7 +75,7 @@ from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name from easybuild.tools.modules import modules_tool, NoModulesTool -from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type +from easybuild.tools.py2vs3 import create_base_metaclass, string_type from easybuild.tools.systemtools import check_os_dependency, pick_dep_version from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA @@ -455,6 +456,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi self.path = path self.rawtxt = read_file(path) self.log.debug("Raw contents from supplied easyconfig file %s: %s", path, self.rawtxt) + if not self.rawtxt.strip(): + raise EasyBuildError('Easyconfig file is empty') else: self.rawtxt = rawtxt self.log.debug("Supplied raw easyconfig contents: %s" % self.rawtxt) @@ -1909,7 +1912,9 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error else: # if no easyblock specified, try to find if one exists if name is None: - name = "UNKNOWN" + if error_on_missing_easyblock: + raise EasyBuildError("No easyblock found as neither name nor easyblock were specified") + return None # The following is a generic way to calculate unique class names for any funny software title class_name = encode_class_name(name) # modulepath will be the namespace + encoded modulename (from the classname) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 9da6e9a2b9..be9bb28bd3 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -90,16 +90,19 @@ # template values which are only generated dynamically TEMPLATE_NAMES_DYNAMIC = [ ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), - ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" - "as specify by the --sysroot configuration option"), - ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), + ('cuda_cc_space_sep_no_period', + "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90')."), ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), + ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), + ('software_commit', "Git commit id to use for the software as specified by --software-commit command line option"), + ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" + "as specified by the --sysroot configuration option"), ] # constant templates that can be used in easyconfigs @@ -128,7 +131,9 @@ ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s', 'download.savannah.gnu.org source url'), ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s', - 'gnu.org source url'), + 'gnu.org source url (ftp mirror)'), + ('GNU_FTP_SOURCE', 'https://ftp.gnu.org/gnu/%(namelower)s', + 'gnu.org source url (main ftp)'), ('GOOGLECODE_SOURCE', 'http://%(namelower)s.googlecode.com/files', 'googlecode.com source url'), ('LAUNCHPAD_SOURCE', 'https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/', @@ -206,6 +211,9 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback template_values['sysroot'] = build_option('sysroot') or '' + # set 'software_commit' template based on 'software_commit' configuration option, default to None + template_values['software_commit'] = build_option('software_commit') or '' + # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: if name in ignore: @@ -363,13 +371,14 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # step 6. CUDA compute capabilities # Use the commandline / easybuild config option if given, else use the value from the EC (as a default) - cuda_compute_capabilities = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities') - if cuda_compute_capabilities: - template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities) - template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) - template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities) - template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities) - sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] + cuda_cc = build_option('cuda_compute_capabilities') or config.get('cuda_compute_capabilities') + if cuda_cc: + template_values['cuda_compute_capabilities'] = ','.join(cuda_cc) + template_values['cuda_cc_space_sep'] = ' '.join(cuda_cc) + template_values['cuda_cc_space_sep_no_period'] = ' '.join(cc.replace('.', '') for cc in cuda_cc) + template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_cc) + template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_cc) + sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_cc] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 6f2a2741ad..cb92b36b76 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -45,6 +45,7 @@ import re import sys import tempfile +from collections import OrderedDict from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR @@ -54,7 +55,7 @@ from easybuild.framework.easyconfig.format.yeb import quote_yaml_special_chars from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.tools import LooseVersion -from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning +from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files @@ -64,7 +65,6 @@ from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.github import fetch_files_from_commit, fetch_files_from_pr from easybuild.tools.multidiff import multidiff -from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import only_if_module_is_available, quote_str @@ -219,10 +219,12 @@ def mk_node_name(spec): if dep in spec['ec'].build_dependencies: dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs) - _dep_graph_dump(dgr, filename) - - if not build_option('silent'): - print("Wrote dependency graph for %d easyconfigs to %s" % (len(specs), filename)) + what = "dependency graph for %d easyconfigs to %s" % (len(specs), filename) + silent = build_option('silent') + if _dep_graph_dump(dgr, filename): + print_msg("Wrote " + what, silent=silent) + else: + print_error("Failed writing " + what, silent=silent) @only_if_module_is_available('pygraph.readwrite.dot', pkgname='python-graph-dot') @@ -232,9 +234,15 @@ def _dep_graph_dump(dgr, filename): dottxt = dot.write(dgr) if os.path.splitext(filename)[-1] == '.dot': # create .dot file - write_file(filename, dottxt) + try: + write_file(filename, dottxt) + except EasyBuildError as e: + print(str(e)) + return False + else: + return True else: - _dep_graph_gv(dottxt, filename) + return _dep_graph_gv(dottxt, filename) @only_if_module_is_available('gv', pkgname='graphviz-python') @@ -242,8 +250,8 @@ def _dep_graph_gv(dottxt, filename): """Render dependency graph to file using graphviz.""" # try and render graph in specified file format gvv = gv.readstring(dottxt) - gv.layout(gvv, 'dot') - gv.render(gvv, os.path.splitext(filename)[-1], filename) + if gv.layout(gvv, 'dot') is not False: + return gv.render(gvv, os.path.splitext(filename)[-1], filename) def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): diff --git a/easybuild/main.py b/easybuild/main.py index 451152f322..b6adc08559 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -727,9 +727,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr if options.ignore_test_failure: raise EasyBuildError("Found both ignore-test-failure and skip-test-step enabled. " "Please use only one of them.") - else: - print_warning("Will not run the test step as requested via skip-test-step. " - "Consider using ignore-test-failure instead and verify the results afterwards") + print_warning("Will not run the test step as requested via skip-test-step. " + "Consider using ignore-test-failure instead and verify the results afterwards") + if options.skip_sanity_check and options.sanity_check_only: + raise EasyBuildError("Found both skip-sanity-check and sanity-check-only enabled. " + "Please use only one of them.") # if EasyStack file is provided, parse it, and loop over the items in the EasyStack file if options.easystack: diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 548ac41187..e9748f3bda 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -78,6 +78,15 @@ class Gcc(Compiler): COMPILER_UNIQUE_OPTION_MAP['strict'] = no_recip_alternative COMPILER_UNIQUE_OPTION_MAP['precise'] = no_recip_alternative + # gcc on RISC-V does not support -mno-recip, -mieee-fp, -mfno-math-errno... + # https://gcc.gnu.org/onlinedocs/gcc/RISC-V-Options.html + # there are no good alternatives, so stick to the default flags + if systemtools.get_cpu_family() == systemtools.RISCV: + COMPILER_UNIQUE_OPTION_MAP['strict'] = [] + COMPILER_UNIQUE_OPTION_MAP['precise'] = [] + COMPILER_UNIQUE_OPTION_MAP['loose'] = ['fno-math-errno'] + COMPILER_UNIQUE_OPTION_MAP['verloose'] = ['fno-math-errno'] + # used when 'optarch' toolchain option is enabled (and --optarch is not specified) COMPILER_OPTIMAL_ARCHITECTURE_OPTION = { (systemtools.AARCH32, systemtools.ARM): 'mcpu=native', # implies -march=native and -mtune=native diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py index 1a21b21943..c33b5ee1a2 100644 --- a/easybuild/toolchains/compiler/intel_compilers.py +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -48,7 +48,8 @@ class IntelCompilers(IntelIccIfort): 'oneapi': (None, "Use oneAPI compilers icx/icpx/ifx instead of classic compilers"), 'oneapi_c_cxx': (None, "Use oneAPI C/C++ compilers icx/icpx instead of classic Intel C/C++ compilers " "(auto-enabled for Intel compilers version 2022.2.0, or newer)"), - 'oneapi_fortran': (False, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler"), + 'oneapi_fortran': (None, "Use oneAPI Fortran compiler ifx instead of classic Intel Fortran compiler " + "(auto-enabled for Intel compilers version 2024.0.0, or newer)"), }) def _set_compiler_vars(self): @@ -75,6 +76,9 @@ def set_variables(self): # auto-enable use of oneAPI C/C++ compilers for sufficiently recent versions of Intel compilers comp_ver = self.get_software_version(self.COMPILER_MODULE_NAME)[0] if LooseVersion(comp_ver) >= LooseVersion('2022.2.0'): + if LooseVersion(comp_ver) >= LooseVersion('2024.0.0'): + if self.options.get('oneapi_fortran', None) is None: + self.options['oneapi_fortran'] = True if self.options.get('oneapi_c_cxx', None) is None: self.options['oneapi_c_cxx'] = True diff --git a/easybuild/toolchains/gmpflf.py b/easybuild/toolchains/gmpflf.py new file mode 100644 index 0000000000..30d10a90f7 --- /dev/null +++ b/easybuild/toolchains/gmpflf.py @@ -0,0 +1,45 @@ +## +# Copyright 2013-2024 Ghent University +# +# This file is triple-licensed under GPLv2 (see below), MIT, and +# BSD three-clause licenses. +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gmpflf compiler toolchain (includes GCC, MPICH, FlexiBLAS, LAPACK, ScaLAPACK and FFTW). + +Authors: + +* Richard Topouchian (University of Bergen) +""" +from easybuild.toolchains.gmpich import Gmpich +from easybuild.toolchains.gfbf import Gfbf +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Gmpflf(Gmpich, FlexiBLAS, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, MPICH, FlexiBLAS, ScaLAPACK and FFTW.""" + NAME = 'gmpflf' + SUBTOOLCHAIN = [Gmpich.NAME, Gfbf.NAME] diff --git a/easybuild/toolchains/linalg/intelmkl.py b/easybuild/toolchains/linalg/intelmkl.py index 8a32d684d9..9dc89da827 100644 --- a/easybuild/toolchains/linalg/intelmkl.py +++ b/easybuild/toolchains/linalg/intelmkl.py @@ -142,6 +142,7 @@ def _set_blas_variables(self): self.variables.nappend_el('CFLAGS', 'DMKL_ILP64') # exact paths/linking statements depend on imkl version + root = self.get_software_root(self.BLAS_MODULE_NAME)[0] found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0] ver = LooseVersion(found_version) if ver < LooseVersion('10.3'): @@ -156,6 +157,8 @@ def _set_blas_variables(self): found_version) else: if ver >= LooseVersion('2021'): + if os.path.islink(os.path.join(root, 'mkl', 'latest')): + found_version = os.readlink(os.path.join(root, 'mkl', 'latest')) basedir = os.path.join('mkl', found_version) else: basedir = 'mkl' diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index 487e6372a3..f5ce1ebc01 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -31,8 +31,8 @@ * Stijn De Weirdt (Ghent University) """ import time +from collections import OrderedDict from easybuild.tools.filetools import det_size -from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.systemtools import get_system_info from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 6bec64764c..2dd71388cd 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -120,6 +120,8 @@ DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' +EASYBUILD_SOURCES_URL = 'https://sources.easybuild.io' +DEFAULT_EXTRA_SOURCE_URLS = (EASYBUILD_SOURCES_URL,) # Filter these CUDA libraries by default from the RPATH sanity check. # These are the only four libraries for which the CUDA toolkit ships stubs. By design, one is supposed to build # against the stub versions, but use the libraries that come with the CUDA driver at runtime. That means they should @@ -259,6 +261,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'rpath_override_dirs', 'required_linked_shared_libs', 'skip', + 'software_commit', 'stop', 'subdir_user_modules', 'sysroot', @@ -307,6 +310,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'set_gid_bit', 'silence_hook_trigger', 'skip_extensions', + 'skip_sanity_check', 'skip_test_cases', 'skip_test_step', 'sticky_bit', @@ -391,6 +395,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'defaultopt': [ 'default_opt_level', ], + DEFAULT_EXTRA_SOURCE_URLS: [ + 'extra_source_urls', + ], DEFAULT_ALLOW_LOADED_MODULES: [ 'allow_loaded_modules', ], diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index b4387454ba..d5b9cd4628 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -40,6 +40,7 @@ import inspect import json import os +from collections import OrderedDict from easybuild.tools import LooseVersion from easybuild.base import fancylogger @@ -60,7 +61,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import read_file from easybuild.tools.modules import modules_tool -from easybuild.tools.py2vs3 import OrderedDict, ascii_lowercase +from easybuild.tools.py2vs3 import ascii_lowercase from easybuild.tools.toolchain.toolchain import DUMMY_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME, is_system_toolchain from easybuild.tools.toolchain.utilities import search_toolchain from easybuild.tools.utilities import INDENT_2SPACES, INDENT_4SPACES @@ -788,7 +789,7 @@ def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False if isinstance(ec, dict): template_values = template_constant_dict(ec) for key in keys: - if '%(' in info[key]: + if info[key] and '%(' in info[key]: try: info[key] = info[key] % template_values except (KeyError, TypeError, ValueError) as err: diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 695043a595..06ca1dddb5 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1242,12 +1242,16 @@ def calc_block_checksum(path, algorithm): return algorithm.hexdigest() -def verify_checksum(path, checksums): +def verify_checksum(path, checksums, computed_checksums=None): """ Verify checksum of specified file. :param path: path of file to verify checksum of - :param checksums: checksum values (and type, optionally, default is MD5), e.g., 'af314', ('sha', '5ec1b') + :param checksums: checksum values to compare to + (and type, optionally, default is MD5), e.g., 'af314', ('sha', '5ec1b') + :param computed_checksums: Optional dictionary of (current) checksum(s) for this file + indexed by the checksum type (e.g. 'sha256'). + Each existing entry will be used, missing ones will be computed. """ filename = os.path.basename(path) @@ -1303,8 +1307,14 @@ def verify_checksum(path, checksums): "2-tuple (type, value), or tuple of alternative checksum specs.", checksum) - actual_checksum = compute_checksum(path, typ) - _log.debug("Computed %s checksum for %s: %s (correct checksum: %s)" % (typ, path, actual_checksum, checksum)) + if computed_checksums is not None and typ in computed_checksums: + actual_checksum = computed_checksums[typ] + computed_str = 'Precomputed' + else: + actual_checksum = compute_checksum(path, typ) + computed_str = 'Computed' + _log.debug("%s %s checksum for %s: %s (correct checksum: %s)" % + (computed_str, typ, path, actual_checksum, checksum)) if actual_checksum != checksum: return False @@ -2409,12 +2419,16 @@ def copy_file(path, target_path, force_in_dry_run=False): try: # check whether path to copy exists (we could be copying a broken symlink, which is supported) path_exists = os.path.exists(path) + # If target is a folder, the target_path will be a file with the same name inside the folder + if os.path.isdir(target_path): + target_path = os.path.join(target_path, os.path.basename(path)) target_exists = os.path.exists(target_path) + if target_exists and path_exists and os.path.samefile(path, target_path): _log.debug("Not copying %s to %s since files are identical", path, target_path) # if target file exists and is owned by someone else than the current user, - # try using shutil.copyfile to just copy the file contents - # since shutil.copy2 will fail when trying to copy over file metadata (since chown requires file ownership) + # copy just the file contents (shutil.copyfile instead of shutil.copy2) + # since copying the file metadata/permissions will fail since chown requires file ownership elif target_exists and os.stat(target_path).st_uid != os.getuid(): shutil.copyfile(path, target_path) _log.info("Copied contents of file %s to %s", path, target_path) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 68ddba632a..f48433e23c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -45,6 +45,7 @@ import sys import tempfile import pwd +from collections import OrderedDict import easybuild.tools.environment as env from easybuild.base import fancylogger # build_log should always stay there, to ensure EasyBuildLog @@ -69,9 +70,9 @@ from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT +from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT -from easybuild.tools.config import DEFAULT_FILTER_RPATH_SANITY_LIBS +from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS @@ -97,7 +98,7 @@ from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes from easybuild.tools.modules import Lmod -from easybuild.tools.py2vs3 import OrderedDict, string_type +from easybuild.tools.py2vs3 import string_type from easybuild.tools.robot import det_robot_path from easybuild.tools.run import run_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes @@ -407,6 +408,8 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), + "extra-source-urls": ("Specify URLs to fetch sources from in addition to those in the easyconfig", + "urltuple", "add_flex", DEFAULT_EXTRA_SOURCE_URLS, {'metavar': 'URL[|URL]'}), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " @@ -438,11 +441,11 @@ def override_options(self): "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), 'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, " "(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None), - 'http-header-fields-urlpat': ("Set extra HTTP header FIELDs when downloading files from URL PATterns. " - "To not log sensitive values, specify a file containing newline separated " - "FIELDs. e.g. \"^https://www.example.com::/path/to/headers.txt\" or " - "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\".", - None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}), + 'http-header-fields-urlpat': ("Set extra HTTP header FIELD when downloading files from URL PATterns. " + "Use FILE (to hide sensitive values) and newline separated FIELDs in the " + "same format. e.g. \"^https://www.example.com::path/to/headers.txt\" or " + "\"client[A-z0-9]*.example.com:: Authorization: Basic token\".", + None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FIELDVALUE|FILE'}), 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False), 'ignore-test-failure': ("Ignore a failing test step", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), @@ -509,8 +512,14 @@ def override_options(self): 'silence-hook-trigger': ("Suppress printing of debug message every time a hook is triggered", None, 'store_true', False), 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), + 'skip-sanity-check': ("Skip running the sanity check step " + "(e.g. testing for installed files or running basic commands)", + None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), + 'software-commit': ( + "Git commit to use for the target software build (robot capabilities are automatically disabled)", + None, 'store', None), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include", None, 'store', None), @@ -656,7 +665,8 @@ def informative_options(self): 'check-conflicts': ("Check for version conflicts in dependency graphs", None, 'store_true', False), 'check-eb-deps': ("Check presence and version of (required and optional) EasyBuild dependencies", None, 'store_true', False), - 'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.'}), + 'dep-graph': ("Create dependency graph. Output format depends on , e.g. 'dot', 'png', 'pdf', 'gv'.", + None, 'store', None, {'metavar': 'depgraph.'}), 'dump-env-script': ("Dump source script to set up build environment based on toolchain/dependencies", None, 'store_true', False), 'last-log': ("Print location to EasyBuild log file of last (failed) session", None, 'store_true', False), @@ -1098,6 +1108,12 @@ def _postprocess_checks(self): else: raise EasyBuildError("Specified sysroot '%s' does not exist!", self.options.sysroot) + # 'software_commit' is specific to a particular software package and cannot be used with 'robot' + if self.options.software_commit: + if self.options.robot is not None: + print_warning("To allow use of --software-commit robot resolution is being disabled") + self.options.robot = None + self.log.info("Checks on configuration options passed") def get_cfg_opt_abs_path(self, opt_name, path): @@ -1590,7 +1606,7 @@ def check_included_multiple(included_easyblocks_from, source): check_included_multiple(included_from_commit, "commit %s" % easyblock_commit) for easyblock in included_from_commit: - print_msg("easyblock %s included from comit %s" % (easyblock, easyblock_commit), log=log) + print_msg("easyblock %s included from commit %s" % (easyblock, easyblock_commit), log=log) include_easyblocks(options.tmpdir, easyblocks_from_commit) diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index b6da8a9245..a8f98480f9 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -32,10 +32,10 @@ * Jørgen Nordmoen (University of Oslo) """ import functools +from collections import OrderedDict from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style -from easybuild.tools.py2vs3 import OrderedDict try: from rich.console import Console, Group diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index f619279060..b52f55a1a2 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -52,7 +52,7 @@ # reload function (built-in in Python 2) -reload = reload +reload = reload # noqa: F821 # string type that can be used in 'isinstance' calls string_type = basestring @@ -89,9 +89,12 @@ def subprocess_terminate(proc, timeout): proc.terminate() +# Wrapped in exec to avoid invalid syntax warnings for Python 3 +exec(''' def raise_with_traceback(exception_class, message, traceback): """Raise exception of specified class with given message and traceback.""" raise exception_class, message, traceback # noqa: E999 +''') def extract_method_name(method_func): diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 73eb657510..24e910fb94 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -42,6 +42,7 @@ import sys import termios import warnings +from collections import OrderedDict from ctypes.util import find_library from socket import gethostname from easybuild.tools.py2vs3 import subprocess_popen_text @@ -64,7 +65,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import IGNORE from easybuild.tools.filetools import is_readable, read_file, which -from easybuild.tools.py2vs3 import OrderedDict, string_type +from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd @@ -735,7 +736,6 @@ def get_os_name(): # platform.linux_distribution was removed in Python 3.8, # see https://docs.python.org/2/library/platform.html#platform.linux_distribution if hasattr(platform, 'linux_distribution'): - # platform.linux_distribution is more useful, but only available since Python 2.6 # this allows to differentiate between Fedora, CentOS, RHEL and Scientific Linux (Rocks is just CentOS) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=PendingDeprecationWarning) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index bbe7f386ad..fb2601805d 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.2') +VERSION = LooseVersion('4.9.3') UNKNOWN = 'UNKNOWN' UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS' diff --git a/eb b/eb index ad14ba61d0..0a510c70ad 100755 --- a/eb +++ b/eb @@ -40,8 +40,8 @@ keyboard_interrupt() { trap keyboard_interrupt SIGINT -# Python 2.6+ or 3.5+ required -REQ_MIN_PY2VER=6 +# Python 2.7+ or 3.5+ required +REQ_MIN_PY2VER=7 REQ_MIN_PY3VER=5 EASYBUILD_MAIN='easybuild.main' diff --git a/requirements.txt b/requirements.txt index 85a9df78e7..d2ce2ccdc1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,12 @@ # keyring is required to provide GitHub token to EasyBuild; # for recent versions of keyring, keyrings.alt must be installed too -keyring +# 19.0 dropped Python 2 support +keyring<19.0; python_version < '3.0' +keyring; python_version >= '3.0' keyrings.alt # GitPython 3.1.15 deprecates Python 3.5 -GitPython==3.1.14; python_version >= '3.0' and python_version < '3.6' +GitPython<3.1.15; python_version >= '3.0' and python_version < '3.6' GitPython; python_version >= '3.6' or python_version <= '3.0' # autopep8 diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 4b63dc605b..c43c84be0e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -38,6 +38,7 @@ import sys import tempfile import textwrap +from collections import OrderedDict from easybuild.tools import LooseVersion from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -72,7 +73,7 @@ from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata -from easybuild.tools.py2vs3 import OrderedDict, reload +from easybuild.tools.py2vs3 import reload from easybuild.tools.robot import det_robot_path, resolve_dependencies from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64 from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext, get_os_name, get_os_version @@ -138,10 +139,13 @@ def tearDown(self): def test_empty(self): """ empty files should not parse! """ + self.assertErrorRegex(EasyBuildError, "expected a valid path", EasyConfig, "") self.contents = "# empty string" self.prep() self.assertRaises(EasyBuildError, EasyConfig, self.eb_file) - self.assertErrorRegex(EasyBuildError, "expected a valid path", EasyConfig, "") + self.contents = "" + self.prep() + self.assertErrorRegex(EasyBuildError, "is empty", EasyConfig, self.eb_file) def test_mandatory(self): """ make sure all checking of mandatory parameters works """ @@ -1447,6 +1451,35 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) + def test_software_commit_template(self): + """Test the %(software_commit)s template""" + + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nconfigopts = "--some-opt=%(software_commit)s"' + test_ec_txt += '\nbuildopts = "--some-opt=%(software_commit)s"' + test_ec_txt += '\ninstallopts = "--some-opt=%(software_commit)s"' + write_file(test_ec, test_ec_txt) + + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=") + self.assertEqual(ec['buildopts'], "--some-opt=") + self.assertEqual(ec['installopts'], "--some-opt=") + + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist + software_commit = '1234bc' + update_build_option('software_commit', software_commit) + + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=%s" % software_commit) + self.assertEqual(ec['buildopts'], "--some-opt=%s" % software_commit) + self.assertEqual(ec['installopts'], "--some-opt=%s" % software_commit) + def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() @@ -1634,6 +1667,10 @@ def test_get_easyblock_class(self): self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY') self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None) + # Test passing neither easyblock nor name + self.assertErrorRegex(EasyBuildError, "neither name nor easyblock were specified", get_easyblock_class, None) + self.assertEqual(get_easyblock_class(None, error_on_missing_easyblock=False), None) + # also test deprecated default_fallback named argument self.assertErrorRegex(EasyBuildError, "DEPRECATED", get_easyblock_class, None, name='gzip', default_fallback=False) @@ -3337,6 +3374,7 @@ def test_template_constant_dict(self): 'nameletter': 'g', 'nameletterlower': 'g', 'parallel': None, + 'software_commit': '', 'sysroot': '', 'toolchain_name': 'foss', 'toolchain_version': '2018a', @@ -3419,6 +3457,7 @@ def test_template_constant_dict(self): 'pyminver': '7', 'pyshortver': '3.7', 'pyver': '3.7.2', + 'software_commit': '', 'sysroot': '', 'version': '0.01', 'version_major': '0', @@ -3484,6 +3523,7 @@ def test_template_constant_dict(self): 'namelower': 'foo', 'nameletter': 'f', 'nameletterlower': 'f', + 'software_commit': '', 'sysroot': '', 'version': '1.2.3', 'version_major': '1', @@ -4606,7 +4646,7 @@ def test_cuda_compute_capabilities(self): toolchain = SYSTEM cuda_compute_capabilities = ['5.1', '7.0', '7.1'] installopts = '%(cuda_compute_capabilities)s' - preinstallopts = '%(cuda_cc_space_sep)s' + preinstallopts = 'period="%(cuda_cc_space_sep)s" noperiod="%(cuda_cc_space_sep_no_period)s"' prebuildopts = '%(cuda_cc_semicolon_sep)s' configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' @@ -4615,7 +4655,7 @@ def test_cuda_compute_capabilities(self): ec = EasyConfig(self.eb_file) self.assertEqual(ec['installopts'], '5.1,7.0,7.1') - self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') + self.assertEqual(ec['preinstallopts'], 'period="5.1 7.0 7.1" noperiod="51 70 71"') self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" ' 'space="sm_51 sm_70 sm_71"') @@ -4625,7 +4665,7 @@ def test_cuda_compute_capabilities(self): init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']}) ec = EasyConfig(self.eb_file) self.assertEqual(ec['installopts'], '4.2,6.3') - self.assertEqual(ec['preinstallopts'], '4.2 6.3') + self.assertEqual(ec['preinstallopts'], 'period="4.2 6.3" noperiod="42 63"') self.assertEqual(ec['prebuildopts'], '4.2;6.3') self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" ' 'space="sm_42 sm_63"') diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 63cd0a93b7..1a0294a02f 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -33,6 +33,7 @@ """ import datetime import glob +import logging import os import re import shutil @@ -297,10 +298,26 @@ def test_checksums(self): 'b7297da8b547d5e74b851d7c4e475900cec4744df0f887ae5c05bf1757c224b4', } + old_log_level = ft._log.getEffectiveLevel() + ft._log.setLevel(logging.DEBUG) # make sure checksums computation/verification is correct for checksum_type, checksum in known_checksums.items(): self.assertEqual(ft.compute_checksum(fp, checksum_type=checksum_type), checksum) - self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum))) + with self.log_to_testlogfile(): + self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum))) + self.assertIn('Computed ' + checksum_type, ft.read_file(self.logfile)) + # Passing precomputed checksums reuses it + with self.log_to_testlogfile(): + computed_checksums = {checksum_type: checksum} + self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum), computed_checksums)) + self.assertIn('Precomputed ' + checksum_type, ft.read_file(self.logfile)) + # If the type isn't contained the checksum will be computed + with self.log_to_testlogfile(): + computed_checksums = {'doesnt exist': 'checksum'} + self.assertTrue(ft.verify_checksum(fp, (checksum_type, checksum), computed_checksums)) + self.assertIn('Computed ' + checksum_type, ft.read_file(self.logfile)) + + ft._log.setLevel(old_log_level) # default checksum type is MD5 self.assertEqual(ft.compute_checksum(fp), known_checksums['md5']) @@ -1839,6 +1856,19 @@ def test_copy_file(self): # printing this message will make test suite fail in Travis/GitHub CI, # since we check for unexpected output produced by the tests print("Skipping overwrite-file-owned-by-other-user copy_file test (%s is missing)" % test_file_to_overwrite) + # Copy a file to a directory owned by some other user, e.g. /tmp (owned by root) + # This might be a common choice for e.g. --copy-ec + target_file_path = tempfile.mktemp("easybuild", dir="/tmp") + test_file_to_copy = os.path.join(self.test_prefix, os.path.basename(target_file_path)) + ft.write_file(test_file_to_copy, test_file_contents) + try: + ft.copy_file(test_file_to_copy, '/tmp') + self.assertEqual(ft.read_file(target_file_path), test_file_contents) + finally: + try: + os.remove(target_file_path) + except FileNotFoundError: + pass # also test behaviour of copy_file under --dry-run build_options = { diff --git a/test/framework/github.py b/test/framework/github.py index c4eb8c8854..d899ad06a1 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -816,18 +816,21 @@ def test_github_det_commit_status(self): res = gh.det_commit_status('easybuilders', GITHUB_REPO, commit_sha, GITHUB_TEST_ACCOUNT) self.assertEqual(res, None) - # recent commit (2023-04-11) with cancelled checks (GitHub Actions only) - commit_sha = 'c074f0bb3110c27d9969c3d0b19dde3eca868bd4' + # recent commit with cancelled checks (GitHub Actions only); + # to update, use https://github.com/easybuilders/easybuild-easyconfigs/actions?query=is%3Acancelled + commit_sha = '52b964c3387d6d6f149ec304f9e23f535e799957' res = gh.det_commit_status('easybuilders', 'easybuild-easyconfigs', commit_sha, GITHUB_TEST_ACCOUNT) self.assertEqual(res, 'cancelled') - # recent commit (2023-04-10) with failing checks (GitHub Actions only) - commit_sha = '1b4a45c62d7deaf19125756c46dc8f011fef66e1' + # recent commit with failing checks (GitHub Actions only) + # to update, use https://github.com/easybuilders/easybuild-easyconfigs/actions?query=is%3Afailure + commit_sha = '85e6c2bbc2fd515a1d4dab607b8d43d0a1ed668f' res = gh.det_commit_status('easybuilders', 'easybuild-easyconfigs', commit_sha, GITHUB_TEST_ACCOUNT) self.assertEqual(res, 'failure') - # recent commit (2023-04-10) with successful checks (GitHub Actions only) - commit_sha = '56812a347acbaaa87f229fe319425020fe399647' + # recent commit with successful checks (GitHub Actions only) + # to update, use https://github.com/easybuilders/easybuild-easyconfigs/actions?query=is%3Asuccess + commit_sha = 'f82a563b8e1f8118c7c3ab23374d0e28e1691fea' res = gh.det_commit_status('easybuilders', 'easybuild-easyconfigs', commit_sha, GITHUB_TEST_ACCOUNT) self.assertEqual(res, 'success') diff --git a/test/framework/modules.py b/test/framework/modules.py index 195bf0339c..8ba4f4ac65 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -55,7 +55,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 110 +TEST_MODULES_COUNT = 111 class ModulesTest(EnhancedTestCase): diff --git a/test/framework/modules/intel-compilers/2024.0.0 b/test/framework/modules/intel-compilers/2024.0.0 new file mode 100644 index 0000000000..a5c0267f9d --- /dev/null +++ b/test/framework/modules/intel-compilers/2024.0.0 @@ -0,0 +1,37 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +Intel C, C++ & Fortran compilers (classic and oneAPI) + + +More information +================ + - Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html + } +} + +module-whatis {Description: Intel C, C++ & Fortran compilers (classic and oneAPI)} +module-whatis {Homepage: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html} +module-whatis {URL: https://software.intel.com/content/www/us/en/develop/tools/oneapi/hpc-toolkit.html} + +set root /tmp/intel-compilers/2024.0.0 + +conflict intel-compilers + +prepend-path CPATH $root/tbb/2021.11/include +prepend-path LD_LIBRARY_PATH $root/compiler/2024.0/linux/lib +prepend-path LD_LIBRARY_PATH $root/tbb/2021.11/lib/intel64/gcc4.8 +prepend-path LIBRARY_PATH $root/compiler/2024.0/linux/lib +prepend-path LIBRARY_PATH $root/tbb/2021.11/lib/intel64/gcc4.8 +prepend-path MANPATH $root/compiler/2024.0/share/man +prepend-path OCL_ICD_FILENAMES $root/compiler/2024.0/lib/libintelocl.so +prepend-path PATH $root/compiler/2024.0/bin +prepend-path TBBROOT $root/tbb/2021.11 +setenv EBROOTINTELMINCOMPILERS "$root" +setenv EBVERSIONINTELMINCOMPILERS "2024.0.0" +setenv EBDEVELINTELMINCOMPILERS "$root/easybuild/Core-intel-compilers-2024.0.0-easybuild-devel" + +# Built with EasyBuild version 4.8.2 diff --git a/test/framework/options.py b/test/framework/options.py index 94c4372f91..2208465db0 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -443,6 +443,27 @@ def test_ignore_test_failure(self): error_pattern = 'Found both ignore-test-failure and skip-test-step enabled' self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) + def test_skip_sanity_check(self): + """Test skipping of sanity check step (--skip-sanity-check).""" + + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, read_file(toy_ec) + "\nsanity_check_commands = ['this_will_fail']") + + args = [test_ec, '--rebuild'] + err_msg = "Sanity check failed" + self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True) + + args.append('--skip-sanity-check') + outtext = self.eb_main(args, do_build=True, raise_error=True) + self.assertNotIn('sanity checking...', outtext) + + # Passing skip and only options is disallowed + args.append('--sanity-check-only') + error_pattern = 'Found both skip-sanity-check and sanity-check-only enabled' + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True) + def test_job(self): """Test submitting build as a job.""" @@ -591,6 +612,8 @@ def run_test(fmt=None): r'^``%\(arch\)s``\s+System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)\s*$', r'^``%\(cuda_cc_space_sep\)s``\s+Space-separated list of CUDA compute capabilities\s*$', r'^``SOURCE_TAR_GZ``\s+Source \.tar\.gz bundle\s+``%\(name\)s-%\(version\)s.tar.gz``\s*$', + r'^``%\(software_commit\)s``\s+Git commit id to use for the software as specified ' + 'by --software-commit command line option', ] else: pattern_lines = [ @@ -603,6 +626,8 @@ def run_test(fmt=None): r'^\s+%\(arch\)s: System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)$', r'^\s+%\(cuda_cc_space_sep\)s: Space-separated list of CUDA compute capabilities$', r'^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)$', + r'^\s+%\(software_commit\)s: Git commit id to use for the software as specified ' + 'by --software-commit command line option', ] for pattern_line in pattern_lines: @@ -2225,7 +2250,7 @@ def test_xxx_include_easyblocks_from_commit(self): import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) - pattern = "== easyblock binary.py included from comit %s" % test_commit + pattern = "== easyblock binary.py included from commit %s" % test_commit self.assertEqual(stderr, '') self.assertIn(pattern, stdout) @@ -6045,7 +6070,7 @@ def test_inject_checksums(self): patterns = [ r"^== injecting sha256 checksums in .*/test\.eb$", r"^== fetching sources & patches for test\.eb\.\.\.$", - r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+\.\.\.$", + r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+$", r"^== injecting sha256 checksums for sources & patches in test\.eb\.\.\.$", r"^== \* toy-0.0\.tar\.gz: %s$" % toy_source_sha256, r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: %s$" % toy_patch_sha256, @@ -6173,7 +6198,7 @@ def test_inject_checksums(self): patterns = [ r"^== injecting md5 checksums in .*/test\.eb$", r"^== fetching sources & patches for test\.eb\.\.\.$", - r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+\.\.\.$", + r"^== backup of easyconfig file saved to .*/test\.eb\.bak_[0-9]+_[0-9]+$", r"^== injecting md5 checksums for sources & patches in test\.eb\.\.\.$", r"^== \* toy-0.0\.tar\.gz: be662daa971a640e40be5c804d9d7d10$", r"^== \* toy-0\.0_fix-silly-typo-in-printf-statement\.patch: a99f2a72cee1689a2f7e3ace0356efb1$", @@ -6813,6 +6838,23 @@ def test_sysroot(self): os.environ['EASYBUILD_SYSROOT'] = doesnotexist self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True) + def test_software_commit(self): + """Test use of --software-commit option.""" + + software_commit = "23be34" + software_commit_arg = '--software-commit=' + software_commit + # Add robot to also test that it gets disabled + stdout, stderr = self._run_mock_eb([software_commit_arg, '--show-config', '--robot'], raise_error=True) + + warning_regex = re.compile(r'.*WARNING:.*--software-commit robot resolution is being disabled.*', re.M) + software_commit_regex = re.compile(r'^software-commit\s*\(C\) = %s$' % software_commit, re.M) + robot_regex = re.compile(r'^robot\s*\(C\) = .*', re.M) + + self.assertTrue(warning_regex.search(stderr), "Pattern '%s' not found in: %s" % (warning_regex, stderr)) + self.assertTrue(software_commit_regex.search(stdout), + "Pattern '%s' not found in: %s" % (software_commit_regex, stdout)) + self.assertFalse(robot_regex.search(stdout), "Pattern '%s' found in: %s" % (robot_regex, stdout)) + def test_accept_eula_for(self): """Test --accept-eula-for configuration option.""" diff --git a/test/framework/repository.py b/test/framework/repository.py index 720e0fa8e8..feb96a13cf 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -41,7 +41,6 @@ from easybuild.tools.filetools import read_file from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.repository.gitrepo import GitRepository -from easybuild.tools.repository.hgrepo import HgRepository from easybuild.tools.repository.svnrepo import SvnRepository from easybuild.tools.repository.repository import init_repository from easybuild.tools.run import run_cmd @@ -129,27 +128,6 @@ def test_svnrepo(self): self.assertExists(os.path.join(repo.wc, 'trunk', 'README.md')) shutil.rmtree(repo.wc) - # this test is disabled because it fails in Travis as a result of bitbucket disabling TLS 1.0/1.1 - # we can consider re-enabling it when moving to a more recent Ubuntu version in the Travis config - # (which implies dropping support for Python 2.6) - # cfr. https://github.com/easybuilders/easybuild-framework/pull/2678 - def DISABLED_test_hgrepo(self): - """Test using HgRepository.""" - # only run this test if pysvn Python module is available - try: - import hglib # noqa - except ImportError: - print("(skipping HgRepository test)") - return - - # GitHub also supports SVN - test_repo_url = 'https://kehoste@bitbucket.org/kehoste/testrepository' - - repo = HgRepository(test_repo_url) - repo.init() - self.assertExists(os.path.join(repo.wc, 'README')) - shutil.rmtree(repo.wc) - def test_init_repository(self): """Test use of init_repository function.""" repo = init_repository('FileRepository', self.path) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 0e53639491..beeb2e3512 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1201,7 +1201,7 @@ def test_fft_env_vars_intel(self): self.assertEqual(tc.get_variable('LIBFFT'), libfft) self.assertEqual(tc.get_variable('LIBFFT_MT'), libfft_mt) - fft_lib_dir = os.path.join(modules.get_software_root('imkl'), 'mkl/2021.4.0/lib/intel64') + fft_lib_dir = os.path.join(modules.get_software_root('imkl'), 'mkl/2021.4/lib/intel64') self.assertEqual(tc.get_variable('FFT_LIB_DIR'), fft_lib_dir) tc = self.get_toolchain('intel', version='2021b') @@ -1321,8 +1321,11 @@ def setup_sandbox_for_intel_fftw(self, moddir, imklver='2018.1.163'): ]) write_file(imkl_fftw_module_path, imkl_fftw_mod_txt) - subdir = 'mkl/%s/lib/intel64' % imklver + # put "latest" symbolic link to short version, used in newer MKL + imklshortver = '.'.join(imklver.split('.')[:2]) + subdir = 'mkl/%s/lib/intel64' % imklshortver os.makedirs(os.path.join(imkl_dir, subdir)) + os.symlink(imklshortver, os.path.join(imkl_dir, 'mkl', 'latest')) for fftlib in mkl_libs: write_file(os.path.join(imkl_dir, subdir, 'lib%s.a' % fftlib), 'foo') subdir = 'lib' @@ -1496,6 +1499,37 @@ def test_intel_toolchain_oneapi(self): self.assertEqual(os.getenv('F90'), 'ifx') self.assertEqual(os.getenv('FC'), 'ifx') + self.modtool.purge() + tc = self.get_toolchain('intel-compilers', version='2024.0.0') + tc.prepare() + + # by default (for version >= 2024.0.0): oneAPI C/C++ compiler + oneAPI Fortran compiler + self.assertEqual(os.getenv('CC'), 'icx') + self.assertEqual(os.getenv('CXX'), 'icpx') + self.assertEqual(os.getenv('F77'), 'ifx') + self.assertEqual(os.getenv('F90'), 'ifx') + self.assertEqual(os.getenv('FC'), 'ifx') + + self.modtool.purge() + tc = self.get_toolchain('intel-compilers', version='2024.0.0') + tc.set_options({'oneapi_fortran': False}) + tc.prepare() + self.assertEqual(os.getenv('CC'), 'icx') + self.assertEqual(os.getenv('CXX'), 'icpx') + self.assertEqual(os.getenv('F77'), 'ifort') + self.assertEqual(os.getenv('F90'), 'ifort') + self.assertEqual(os.getenv('FC'), 'ifort') + + self.modtool.purge() + tc = self.get_toolchain('intel-compilers', version='2024.0.0') + tc.set_options({'oneapi_c_cxx': False, 'oneapi_fortran': False}) + tc.prepare() + self.assertEqual(os.getenv('CC'), 'icc') + self.assertEqual(os.getenv('CXX'), 'icpc') + self.assertEqual(os.getenv('F77'), 'ifort') + self.assertEqual(os.getenv('F90'), 'ifort') + self.assertEqual(os.getenv('FC'), 'ifort') + self.modtool.purge() tc = self.get_toolchain('intel', version='2021b') tc.set_options({'oneapi_c_cxx': True}) @@ -2203,7 +2237,8 @@ def test_compiler_cache(self): "#!/bin/bash", "echo 'This is a %s wrapper'" % cache_tool, "NAME=${0##*/}", - "comm=$(which -a $NAME | sed 1d)", + "comms=($(which -a $NAME))", + "comm=${comms[1]}", # First entry is this wrapper, take 2nd "$comm $@", "exit 0" ]