diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 1fd59b932d..7583158ee6 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -42,6 +42,7 @@ import copy import glob import inspect +import json import os import re import stat @@ -67,7 +68,7 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning -from easybuild.tools.config import DEFAULT_ENVVAR_USERS_MODULES +from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths @@ -156,6 +157,7 @@ def __init__(self, ec): self.patches = [] self.src = [] self.checksums = [] + self.json_checksums = None # build/install directories self.builddir = None @@ -347,23 +349,55 @@ def get_checksum_for(self, checksums, filename=None, index=None): Obtain checksum for given filename. :param checksums: a list or tuple of checksums (or None) - :param filename: name of the file to obtain checksum for (Deprecated) + :param filename: name of the file to obtain checksum for :param index: index of file in list """ - # Filename has never been used; flag it as deprecated - if filename: - self.log.deprecated("Filename argument to get_checksum_for() is deprecated", '5.0') + checksum = None + + # sometimes, filename are specified as a dict + if isinstance(filename, dict): + filename = filename['filename'] # if checksums are provided as a dict, lookup by source filename as key - if isinstance(checksums, (list, tuple)): + if isinstance(checksums, dict): + if filename is not None and filename in checksums: + checksum = checksums[filename] + else: + checksum = None + elif isinstance(checksums, (list, tuple)): if index is not None and index < len(checksums) and (index >= 0 or abs(index) <= len(checksums)): - return checksums[index] + checksum = checksums[index] else: - return None + checksum = None elif checksums is None: - return None + checksum = None + else: + raise EasyBuildError("Invalid type for checksums (%s), should be dict, list, tuple or None.", + type(checksums)) + + if checksum is None or build_option("checksum_priority") == CHECKSUM_PRIORITY_JSON: + json_checksums = self.get_checksums_from_json() + return json_checksums.get(filename, None) else: - raise EasyBuildError("Invalid type for checksums (%s), should be list, tuple or None.", type(checksums)) + return checksum + + def get_checksums_from_json(self, always_read=False): + """ + Get checksums for this software that are provided in a checksums.json file + + :param: always_read: always read the checksums.json file, even if it has been read before + """ + if always_read or self.json_checksums is None: + try: + path = self.obtain_file("checksums.json", no_download=True) + self.log.info("Loading checksums from file %s", path) + json_txt = read_file(path) + self.json_checksums = json.loads(json_txt) + # if the file can't be found, return an empty dict + except EasyBuildError: + self.json_checksums = {} + + return self.json_checksums def fetch_source(self, source, checksum=None, extension=False, download_instructions=None): """ @@ -445,7 +479,8 @@ def fetch_sources(self, sources=None, checksums=None): if source is None: raise EasyBuildError("Empty source in sources list at index %d", index) - src_spec = self.fetch_source(source, self.get_checksum_for(checksums=checksums, index=index)) + checksum = self.get_checksum_for(checksums=checksums, filename=source, index=index) + src_spec = self.fetch_source(source, checksum=checksum) if src_spec: self.src.append(src_spec) else: @@ -477,7 +512,7 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): if path: self.log.debug('File %s found for patch %s', path, patch_spec) patch_info['path'] = path - patch_info['checksum'] = self.get_checksum_for(checksums, index=index) + patch_info['checksum'] = self.get_checksum_for(checksums, filename=patch_info['name'], index=index) if extension: patches.append(patch_info) @@ -638,7 +673,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): # verify checksum (if provided) self.log.debug('Verifying checksums for extension source...') - fn_checksum = self.get_checksum_for(checksums, index=0) + fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0) if verify_checksum(src_path, fn_checksum): self.log.info('Checksum for extension source %s verified', src_fn) elif build_option('ignore_checksums'): @@ -672,7 +707,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): patch = patch['path'] patch_fn = os.path.basename(patch) - checksum = self.get_checksum_for(checksums[1:], index=idx) + checksum = self.get_checksum_for(checksums, filename=patch_fn, index=idx+1) if verify_checksum(patch, checksum): self.log.info('Checksum for extension patch %s verified', patch_fn) elif build_option('ignore_checksums'): @@ -694,7 +729,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): return exts_sources def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, - git_config=None, download_instructions=None, alt_location=None): + git_config=None, no_download=False, download_instructions=None, alt_location=None): """ Locate the file with the given name - searches in different subdirectories of source path @@ -705,6 +740,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No :param download_filename: filename with which the file should be downloaded, and then renamed to :param force_download: always try to download file, even if it's already available in source path :param git_config: dictionary to define how to download a git repository + :param no_download: do not try to download the file :param download_instructions: instructions to manually add source (used for complex cases) :param alt_location: alternative location to use instead of self.name """ @@ -818,6 +854,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if self.dry_run: self.dry_run_msg(" * %s found at %s", filename, foundfile) return foundfile + elif no_download: + if self.dry_run: + self.dry_run_msg(" * %s (MISSING)", filename) + return filename + else: + raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... " + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) elif git_config: return get_source_tarball_from_git(filename, targetdir, git_config) else: @@ -2280,7 +2323,7 @@ def fetch_step(self, skip_checksums=False): # fetch patches if self.cfg['patches'] + self.cfg['postinstallpatches']: - if isinstance(self.cfg['checksums'], (list, tuple)): + if self.cfg['checksums'] and isinstance(self.cfg['checksums'], (list, tuple)): # if checksums are provided as a list, first entries are assumed to be for sources patches_checksums = self.cfg['checksums'][len(self.cfg['sources']):] else: @@ -2367,6 +2410,20 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): patches = ent.get('patches', []) checksums = ent.get('checksums', []) + if not checksums: + checksums_from_json = self.get_checksums_from_json() + # recreate a list of checksums. If each filename is found, the generated list of checksums should match + # what is expected in list format + for fn in sources + patches: + # if the filename is a tuple, the actual source file name is the first element + if isinstance(fn, tuple): + fn = fn[0] + # if the filename is a dict, the actual source file name is the "filename" element + if isinstance(fn, dict): + fn = fn["filename"] + if fn in checksums_from_json.keys(): + checksums += [checksums_from_json[fn]] + if source_cnt is None: source_cnt = len(sources) patch_cnt, checksum_cnt = len(patches), len(checksums) @@ -4406,6 +4463,67 @@ class StopException(Exception): pass +def inject_checksums_to_json(ecs, checksum_type): + """ + Inject checksums of given type in corresponding json files + + :param ecs: list of EasyConfig instances to calculate checksums and inject them into checksums.json + :param checksum_type: type of checksum to use + """ + for ec in ecs: + ec_fn = os.path.basename(ec['spec']) + ec_dir = os.path.dirname(ec['spec']) + print_msg("injecting %s checksums for %s in checksums.json" % (checksum_type, ec['spec']), log=_log) + + # get easyblock instance and make sure all sources/patches are available by running fetch_step + print_msg("fetching sources & patches for %s..." % ec_fn, log=_log) + app = get_easyblock_instance(ec) + app.update_config_template_run_step() + app.fetch_step(skip_checksums=True) + + # compute & inject checksums for sources/patches + print_msg("computing %s checksums for sources & patches for %s..." % (checksum_type, ec_fn), log=_log) + checksums = {} + for entry in app.src + app.patches: + checksum = compute_checksum(entry['path'], checksum_type) + print_msg("* %s: %s" % (os.path.basename(entry['path']), checksum), log=_log) + checksums[os.path.basename(entry['path'])] = checksum + + # compute & inject checksums for extension sources/patches + if app.exts: + print_msg("computing %s checksums for extensions for %s..." % (checksum_type, ec_fn), log=_log) + + for ext in app.exts: + # compute checksums for extension sources & patches + if 'src' in ext: + src_fn = os.path.basename(ext['src']) + checksum = compute_checksum(ext['src'], checksum_type) + print_msg(" * %s: %s" % (src_fn, checksum), log=_log) + checksums[src_fn] = checksum + for ext_patch in ext.get('patches', []): + patch_fn = os.path.basename(ext_patch['path']) + checksum = compute_checksum(ext_patch['path'], checksum_type) + print_msg(" * %s: %s" % (patch_fn, checksum), log=_log) + checksums[patch_fn] = checksum + + # actually inject new checksums or overwrite existing ones (if --force) + existing_checksums = app.get_checksums_from_json(always_read=True) + for filename in checksums: + if filename not in existing_checksums: + existing_checksums[filename] = checksums[filename] + # don't do anything if the checksum already exist and is the same + elif checksums[filename] != existing_checksums[filename]: + if build_option('force'): + print_warning("Found existing checksums for %s, overwriting them (due to --force)..." % ec_fn) + existing_checksums[filename] = checksums[filename] + else: + raise EasyBuildError("Found existing checksum for %s, use --force to overwrite them" % filename) + + # actually write the checksums + with open(os.path.join(ec_dir, 'checksums.json'), 'w') as outfile: + json.dump(existing_checksums, outfile, indent=2, sort_keys=True) + + def inject_checksums(ecs, checksum_type): """ Inject checksums of given type in specified easyconfig files diff --git a/easybuild/main.py b/easybuild/main.py index b4de2480ea..d6d07ef2cf 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -34,6 +34,7 @@ :author: Toon Willems (Ghent University) :author: Ward Poelmans (Ghent University) :author: Fotis Georgatos (Uni.Lu, NTUA) +:author: Maxime Boissonneault (Compute Canada) """ import copy import os @@ -45,7 +46,7 @@ # expect missing log output when this not the case! from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning, stop_logging -from easybuild.framework.easyblock import build_and_install_one, inject_checksums +from easybuild.framework.easyblock import build_and_install_one, inject_checksums, inject_checksums_to_json from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR from easybuild.framework.easystack import parse_easystack from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs @@ -425,7 +426,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files - easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums) + validate = not options.inject_checksums and not options.inject_checksums_to_json + easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=validate) # handle --check-contrib & --check-style options if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style): @@ -453,6 +455,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options or options.copy_ec keep_available_modules = keep_available_modules or options.inject_checksums or options.sanity_check_only + keep_available_modules = keep_available_modules or options.inject_checksums_to_json # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not keep_available_modules: @@ -538,8 +541,12 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): with rich_live_cm(): inject_checksums(ordered_ecs, options.inject_checksums) + elif options.inject_checksums_to_json: + inject_checksums_to_json(ordered_ecs, options.inject_checksums_to_json) + # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] + stop_options += [options.inject_checksums_to_json] if any(no_ec_opts) or any(stop_options): clean_exit(logfile, eb_tmpdir, testing) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 96daf4c25c..a5b8f65001 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -34,6 +34,7 @@ :author: Ward Poelmans (Ghent University) :author: Damian Alvarez (Forschungszentrum Juelich GmbH) :author: Andy Georges (Ghent University) +:author: Maxime Boissonneault (Compute Canada) """ import copy import glob @@ -126,6 +127,11 @@ FORCE_DOWNLOAD_CHOICES = [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES] DEFAULT_FORCE_DOWNLOAD = FORCE_DOWNLOAD_SOURCES +CHECKSUM_PRIORITY_JSON = "json" +CHECKSUM_PRIORITY_EASYCONFIG = "easyconfig" +CHECKSUM_PRIORITY_CHOICES = [CHECKSUM_PRIORITY_JSON, CHECKSUM_PRIORITY_EASYCONFIG] +DEFAULT_CHECKSUM_PRIORITY = CHECKSUM_PRIORITY_EASYCONFIG + # package name for generic easyblocks GENERIC_EASYBLOCK_PKG = 'generic' @@ -180,6 +186,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'accept_eula_for', 'aggregate_regtest', 'backup_modules', + 'checksum_priority', 'container_config', 'container_image_format', 'container_image_name', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6128c3e650..3220f4def2 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -33,6 +33,7 @@ :author: Toon Willems (Ghent University) :author: Ward Poelmans (Ghent University) :author: Damian Alvarez (Forschungszentrum Juelich GmbH) +:author: Maxime Boissonneault (Compute Canada) """ import copy import glob @@ -59,6 +60,7 @@ from easybuild.tools import build_log, run # build_log should always stay there, to ensure EasyBuildLog from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror +from easybuild.tools.config import CHECKSUM_PRIORITY_CHOICES, DEFAULT_CHECKSUM_PRIORITY from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENV_FOR_SHEBANG, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE, DEFAULT_JOB_BACKEND @@ -359,6 +361,9 @@ def override_options(self): 'check-ebroot-env-vars': ("Action to take when defined $EBROOT* environment variables are found " "for which there is no matching loaded module; " "supported values: %s" % ', '.join(EBROOT_ENV_VAR_ACTIONS), None, 'store', WARN), + 'checksum-priority': ("When checksums are found in both the EasyConfig and the checksums.json file" + "Define which one to use. ", + 'choice', 'store_or_None', DEFAULT_CHECKSUM_PRIORITY, CHECKSUM_PRIORITY_CHOICES), 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'cleanup-tmpdir': ("Cleanup tmp dir after successful run.", None, 'store_true', True), 'color': ("Colorize output", 'choice', 'store', fancylogger.Colorize.AUTO, fancylogger.Colorize, @@ -783,6 +788,8 @@ def easyconfig_options(self): int, 'store', DEFAULT_INDEX_MAX_AGE), 'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)", 'choice', 'store_or_None', CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES), + 'inject-checksums-to-json': ("Inject checksums of specified type for sources/patches into checksums.json", + 'choice', 'store_or_None', CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES), 'local-var-naming-check': ("Mode to use when checking whether local variables follow the recommended " "naming scheme ('log': only log warnings (no printed messages); 'warn': print " "warnings; 'error': fail with an error)", 'choice', 'store', @@ -1168,8 +1175,8 @@ def _postprocess_config(self): self.options.ignore_osdeps = True self.options.modules_tool = None - # imply --disable-pre-create-installdir with --inject-checksums - if self.options.inject_checksums: + # imply --disable-pre-create-installdir with --inject-checksums or --inject-checksums-to-json + if self.options.inject_checksums or self.options.inject_checksums_to_json: self.options.pre_create_installdir = False def _postprocess_list_avail(self): diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 3a218160bd..2174a2b0da 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -27,6 +27,7 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) +@author: Maxime Boissonneault (Compute Canada) """ import os import re @@ -1697,8 +1698,14 @@ def test_obtain_file(self): res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball)) + # test no_download option + urls = ['file://%s' % tmpdir_subdir] + error_pattern = "Couldn't find file toy-0.0.tar.gz anywhere, and downloading it is disabled" + self.assertErrorRegex(EasyBuildError, error_pattern, eb.obtain_file, + toy_tarball, urls=urls, alt_location='alt_toy', no_download=True) + # 'downloading' a file to (first) alternative sourcepath works - res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir], alt_location='alt_toy') + res = eb.obtain_file(toy_tarball, urls=urls, alt_location='alt_toy') self.assertEqual(res, os.path.join(tmpdir, 'a', 'alt_toy', toy_tarball)) # make sure that directory in which easyconfig file is located is *ignored* when alt_location is used @@ -2346,6 +2353,64 @@ def test_checksum_step(self): self.mock_stderr(False) self.disallow_deprecated_behaviour() + # create test easyconfig from which checksums have been stripped + test_ec = os.path.join(self.test_prefix, 'test.eb') + ectxt = read_file(toy_ec) + regex = re.compile(r"'?checksums'?\s*[=:]\s*\[[^]]+\].*", re.M) + ectxt = regex.sub('', ectxt) + write_file(test_ec, ectxt) + + ec_json = process_easyconfig(test_ec)[0] + + # make sure that test easyconfig file indeed doesn't contain any checksums (either top-level or for extensions) + self.assertEqual(ec_json['ec']['checksums'], []) + for ext in ec_json['ec']['exts_list']: + if isinstance(ext, string_type): + continue + elif isinstance(ext, tuple): + self.assertEqual(ext[2].get('checksums', []), []) + else: + self.assertTrue(False, "Incorrect extension type: %s" % type(ext)) + + # put checksums.json in place next to easyconfig file being used for the tests + toy_checksums_json = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'checksums.json') + copy_file(toy_checksums_json, os.path.join(self.test_prefix, 'checksums.json')) + + # test without checksums, it should work since they are in checksums.json + eb_json = get_easyblock_instance(ec_json) + eb_json.fetch_sources() + eb_json.checksum_step() + + # if we look only at checksums in Json, it should succeed + eb = get_easyblock_instance(ec) + build_options = { + 'checksum_priority': config.CHECKSUM_PRIORITY_JSON + } + init_config(build_options=build_options) + eb.fetch_sources() + eb.checksum_step() + + # if we look only at (incorrect) checksums in easyconfig, it should fail + eb = get_easyblock_instance(ec) + build_options = { + 'checksum_priority': config.CHECKSUM_PRIORITY_EASYCONFIG + } + init_config(build_options=build_options) + eb.fetch_sources() + error_msg = "Checksum verification for .*/toy-0.0.tar.gz using .* failed" + self.assertErrorRegex(EasyBuildError, error_msg, eb.checksum_step) + + # also check verification of checksums for extensions, which is part of fetch_extension_sources + error_msg = "Checksum verification for extension source bar-0.0.tar.gz failed" + self.assertErrorRegex(EasyBuildError, error_msg, eb.collect_exts_file_info) + + # also check with deprecated fetch_extension_sources method + self.allow_deprecated_behaviour() + self.mock_stderr(True) + self.assertErrorRegex(EasyBuildError, error_msg, eb.fetch_extension_sources) + self.mock_stderr(False) + self.disallow_deprecated_behaviour() + # if --ignore-checksums is enabled, faulty checksums are reported but otherwise ignored (no error) build_options = { 'ignore_checksums': True, @@ -2467,6 +2532,23 @@ def run_checks(): eb.cfg['sources'] = [{'filename': 'toy-0.0.tar.gz', 'download_fileame': 'toy.tar.gz'}] self.assertEqual(eb.check_checksums(), []) + # no checksums in easyconfig, then picked up from checksums.json next to easyconfig file + test_ec = os.path.join(self.test_prefix, 'test.eb') + copy_file(toy_ec, test_ec) + ec = process_easyconfig(test_ec)[0] + eb = get_easyblock_instance(ec) + eb.cfg['checksums'] = [] + res = eb.check_checksums() + self.assertEqual(len(res), 1) + expected = "Checksums missing for one or more sources/patches in test.eb: " + expected += "found 1 sources + 2 patches vs 0 checksums" + self.assertEqual(res[0], expected) + + # all is fine is checksums.json is also copied + copy_file(os.path.join(os.path.dirname(toy_ec), 'checksums.json'), self.test_prefix) + eb.json_checksums = None + self.assertEqual(eb.check_checksums(), []) + def test_this_is_easybuild(self): """Test 'this_is_easybuild' function (and get_git_revision function used by it).""" # make sure both return a non-Unicode string diff --git a/test/framework/easyconfigs/test_ecs/t/toy/checksums.json b/test/framework/easyconfigs/test_ecs/t/toy/checksums.json new file mode 100644 index 0000000000..5449a3e2a2 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/t/toy/checksums.json @@ -0,0 +1,10 @@ +{ + "bar-0.0_fix-silly-typo-in-printf-statement.patch": "84db53592e882b5af077976257f9c7537ed971cb2059003fd4faa05d02cae0ab", + "bar-0.0_fix-very-silly-typo-in-printf-statement.patch": "d0bf102f9c5878445178c5f49b7cd7546e704c33fe2060c7354b7e473cfeb52b", + "bar-0.0.tar.gz": "f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7", + "barbar-0.0.tar.gz": "d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838", + "bar.tgz": "33ac60685a3e29538db5094259ea85c15906cbd0f74368733f4111eab6187c8f", + "toy-extra.txt": "4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458", + "toy-0.0.tar.gz": "44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc", + "toy-0.0_fix-silly-typo-in-printf-statement.patch": "81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487" +} diff --git a/test/framework/filetools.py b/test/framework/filetools.py index b9d4de28b9..07d64ad015 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2384,7 +2384,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 89) + self.assertEqual(len(index), 90) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), @@ -2395,7 +2395,7 @@ def test_index_functions(self): self.assertTrue(fn in index) for fp in index: - self.assertTrue(fp.endswith('.eb')) + self.assertTrue(fp.endswith('.eb') or os.path.basename(fp) == 'checksums.json') # set up some files to create actual index file for ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g')) diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 72a21a73ed..0adcaa5f5c 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -1105,6 +1105,8 @@ def test_module_naming_scheme(self): ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(ecs_dir) for fil in files] + # keep only easyconfig files (there may be additional files like patches, checksums.json, etc.) + ec_files = [x for x in ec_files if x.endswith('.eb')] build_options = { 'check_osdeps': False, diff --git a/test/framework/options.py b/test/framework/options.py index b541c024ae..a64bd2749b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -28,6 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ import glob +import json import os import re import shutil @@ -5868,6 +5869,36 @@ def test_inject_checksums(self): self.assertEqual(stdout, '') self.assertTrue("option --inject-checksums: invalid choice" in stderr) + def test_inject_checksums_to_json(self): + """Test --inject-checksums-to-json.""" + + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') + copy_file(toy_ec, test_ec) + test_ec_txt = read_file(test_ec) + + args = [test_ec, '--inject-checksums-to-json'] + self._run_mock_eb(args, raise_error=True, strip=True) + + self.assertEqual(test_ec_txt, read_file(test_ec)) + + checksums_json_txt = read_file(os.path.join(self.test_prefix, 'checksums.json')) + expected_dict = { + 'bar-0.0.tar.gz': 'f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7', + 'bar-0.0_fix-silly-typo-in-printf-statement.patch': + '84db53592e882b5af077976257f9c7537ed971cb2059003fd4faa05d02cae0ab', + 'bar-0.0_fix-very-silly-typo-in-printf-statement.patch': + 'd0bf102f9c5878445178c5f49b7cd7546e704c33fe2060c7354b7e473cfeb52b', + 'bar.tgz': '33ac60685a3e29538db5094259ea85c15906cbd0f74368733f4111eab6187c8f', + 'barbar-0.0.tar.gz': 'd5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838', + 'toy-0.0.tar.gz': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', + 'toy-0.0_fix-silly-typo-in-printf-statement.patch': + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487', + 'toy-extra.txt': '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458', + } + self.assertEqual(json.loads(checksums_json_txt), expected_dict) + def test_force_download(self): """Test --force-download""" topdir = os.path.dirname(os.path.abspath(__file__)) @@ -5903,6 +5934,9 @@ def test_enforce_checksums(self): toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') + # wipe $EASYBUILD_ROBOT_PATHS to avoid that checksums.json for toy is found in test_ecs + del os.environ['EASYBUILD_ROBOT_PATHS'] + args = [ test_ec, '--stop=source',