Merge pull request #4578 from Flamefire/fix-checksums-verification-5

Fix the checksum type check

easybuild/framework/easyconfig/types.py

@@ -613,19 +613,39 @@ def ensure_iterable_license_specs(specs):
 }))
 # checksums is a list of checksums, one entry per file (source/patch)
 # each entry can be:
+# None
 # a single checksum value (string)
 # a single checksum value of a specified type (2-tuple, 1st element is checksum type, 2nd element is checksum)
 # a list of checksums (of different types, perhaps different formats), which should *all* be valid
-# a dictionary with a mapping from filename to checksum value
-CHECKSUM_LIST = (list, as_hashable({'elem_types': [str, tuple, STRING_DICT]}))
-CHECKSUMS = (list, as_hashable({'elem_types': [str, tuple, STRING_DICT, CHECKSUM_LIST]}))
+# a tuple of checksums (of different types, perhaps different formats), where one should be valid
+# a dictionary with a mapping from filename to checksum (None, value, type&value, alternatives)
+
+# Type & value, value may be an int for type "size"
+# This is a bit too permissive as it allows the first element to be an int and doesn't restrict the number of elements
+CHECKSUM_AND_TYPE = (tuple, as_hashable({'elem_types': [str, int]}))
+CHECKSUM_LIST = (list, as_hashable({'elem_types': [str, CHECKSUM_AND_TYPE]}))
+CHECKSUM_TUPLE = (tuple, as_hashable({'elem_types': [str, CHECKSUM_AND_TYPE]}))
+CHECKSUM_DICT = (dict, as_hashable(
+    {
+        'elem_types': [type(None), str, CHECKSUM_AND_TYPE, CHECKSUM_TUPLE, CHECKSUM_LIST],
+        'key_types': [str],
+    }
+))
+# At the top-level we allow tuples/lists containing a dict
+CHECKSUM_LIST_W_DICT = (list, as_hashable({'elem_types': [str, CHECKSUM_AND_TYPE, CHECKSUM_DICT]}))
+CHECKSUM_TUPLE_W_DICT = (tuple, as_hashable({'elem_types': [str, CHECKSUM_AND_TYPE, CHECKSUM_DICT]}))
+
+CHECKSUMS = (list, as_hashable({'elem_types': [type(None), str, CHECKSUM_AND_TYPE,
+                                               CHECKSUM_LIST_W_DICT, CHECKSUM_TUPLE_W_DICT, CHECKSUM_DICT]}))
 
-CHECKABLE_TYPES = [CHECKSUM_LIST, CHECKSUMS, DEPENDENCIES, DEPENDENCY_DICT, LIST_OF_STRINGS,
+CHECKABLE_TYPES = [CHECKSUM_AND_TYPE, CHECKSUM_LIST, CHECKSUM_TUPLE,
+                   CHECKSUM_LIST_W_DICT, CHECKSUM_TUPLE_W_DICT, CHECKSUM_DICT, CHECKSUMS,
+                   DEPENDENCIES, DEPENDENCY_DICT, LIST_OF_STRINGS,
                    SANITY_CHECK_PATHS_DICT, SANITY_CHECK_PATHS_ENTRY, STRING_DICT, STRING_OR_TUPLE_LIST,
                    STRING_OR_TUPLE_DICT, STRING_OR_TUPLE_OR_DICT_LIST, TOOLCHAIN_DICT, TUPLE_OF_STRINGS]
 
 # easy types, that can be verified with isinstance
-EASY_TYPES = [str, bool, dict, int, list, str, tuple]
+EASY_TYPES = [str, bool, dict, int, list, str, tuple, type(None)]
 
 # type checking is skipped for easyconfig parameters names not listed in PARAMETER_TYPES
 PARAMETER_TYPES = {

test/framework/type_checking.py

@@ -174,10 +174,12 @@ def test_check_type_of_param_value_sanity_check_paths(self):
     def test_check_type_of_param_value_checksums(self):
         """Test check_type_of_param_value function for checksums."""
 
-        md5_checksum = 'fa618be8435447a017fd1bf2c7ae9224'
-        sha256_checksum1 = 'fa618be8435447a017fd1bf2c7ae922d0428056cfc7449f7a8641edf76b48265'
-        sha256_checksum2 = 'b5f9cb06105c1d2d30719db5ffb3ea67da60919fb68deaefa583deccd8813551'
-        sha256_checksum3 = '033be54514a03e255df75c5aee8f9e672f663f93abb723444caec8fe43437bde'
+        # Using (actually invalid) prefix to better detect those in case of errors
+        md5_checksum = 'md518be8435447a017fd1bf2c7ae9224'
+        sha256_checksum1 = 'sha18be8435447a017fd1bf2c7ae922d0428056cfc7449f7a8641edf76b48265'
+        sha256_checksum2 = 'sha2cb06105c1d2d30719db5ffb3ea67da60919fb68deaefa583deccd8813551'
+        sha256_checksum3 = 'sha3e54514a03e255df75c5aee8f9e672f663f93abb723444caec8fe43437bde'
+        filesize = 45617379
 
         # valid values for 'checksums' easyconfig parameters
         inputs = [
@@ -190,6 +192,7 @@ def test_check_type_of_param_value_checksums(self):
             # one checksum of specific type (as 2-tuple)
             [('md5', md5_checksum)],
             [('sha256', sha256_checksum1)],
+            [('size', filesize)],
             # alternative checksums for a single file (n-tuple)
             [(sha256_checksum1, sha256_checksum2)],
             [(sha256_checksum1, sha256_checksum2, sha256_checksum3)],
@@ -213,17 +216,33 @@ def test_check_type_of_param_value_checksums(self):
                 # two checksums for a single file, *both* should match
                 [sha256_checksum1, md5_checksum],
                 # three checksums for a single file, *all* should match
-                [sha256_checksum1, ('md5', md5_checksum), {'foo.txt': sha256_checksum1}],
+                [sha256_checksum1, ('md5', md5_checksum), ('size', filesize)],
                 # single checksum for a single file
                 sha256_checksum1,
                 # filename-to-checksum mapping
-                {'foo.txt': sha256_checksum1, 'bar.txt': sha256_checksum2},
+                {'foo.txt': sha256_checksum1, 'bar.txt': sha256_checksum2, 'baz.txt': ('size', filesize)},
                 # 3 alternative checksums for a single file, one match is sufficient
                 (sha256_checksum1, sha256_checksum2, sha256_checksum3),
-            ]
+                # two alternative checksums for a single file (not to be confused by checksum-type & -value tuple)
+                (sha256_checksum1, md5_checksum),
+                # three alternative checksums for a single file of different types
+                (sha256_checksum1, ('md5', md5_checksum), ('size', filesize)),
+                # alternative checksums in dicts are also allowed
+                {'foo.txt': (sha256_checksum2, sha256_checksum3), 'bar.txt': (sha256_checksum1, md5_checksum)},
+                # Same but with lists -> all must match for each file
+                {'foo.txt': [sha256_checksum2, sha256_checksum3], 'bar.txt': [sha256_checksum1, md5_checksum]},
+            ],
+            # None is allowed, meaning skip the checksum
+            [
+                None,
+                # Also in mappings
+                {'foo.txt': sha256_checksum1, 'bar.txt': None},
+            ],
         ]
         for inp in inputs:
-            self.assertEqual(check_type_of_param_value('checksums', inp), (True, inp))
+            type_ok, newval = check_type_of_param_value('checksums', inp)
+            self.assertIs(type_ok, True, 'Failed for ' + str(inp))
+            self.assertEqual(newval, inp)
 
     def test_check_type_of_param_value_patches(self):
         """Test check_type_of_param_value function for patches."""