From 123f7bce7c1f5c67040f068614cc3177c8a27eff Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Fri, 18 Jan 2019 10:14:52 +0100 Subject: [PATCH] fix issue checksum not constant when using git_config (#2726) Replace 'tar' by 'git archive' as it seems more reproducible. Use 'gzip --no-name' to omit timestamps. --- easybuild/tools/filetools.py | 18 +++++++++++++++--- test/framework/filetools.py | 26 ++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index fba94dfa02..0de63c4c00 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1787,9 +1787,21 @@ def get_source_tarball_from_git(filename, targetdir, git_config): run.run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) - # create an archive and delete the git repo directory - tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] - run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False) + # create a git archive (in tar format, gzip later) + git_cmd = ['git', 'archive', '-o', targetpath[:-3], '--prefix=%s/' % (repo_name), 'HEAD'] + run.run_cmd(' '.join(git_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) + + # git archive does not automatically recurse into submodules + if recursive: + # create a git archive for each submdule, and append it to the target + git_cmd = ['git', 'submodule', 'foreach', "'git archive -o $name.tar --prefix=%s/$path/ HEAD'" % (repo_name)] + tar_cmd = ['git', 'submodule', 'foreach', "'tar -f %s --concatenate $name.tar'" % (targetpath[:-3])] + run.run_cmd(' '.join(git_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) + run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) + + # compress (concatenated) tarball, use gzip --no-name option (omitting timestamps makes the checksum constant) + gzip_cmd = ['gzip', '-nf', targetpath[:-3]] + run.run_cmd(' '.join(gzip_cmd), log_all=True, log_ok=False, simple=False, regexp=False) # cleanup (repo_name dir does not exist in dry run mode) change_dir(cwd) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index d055a0860a..c87e17ce05 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1749,7 +1749,9 @@ def run_check(): expected = '\n'.join([ ' running command "git clone --branch master git@github.com:hpcugent/testrepository.git"', " \(in .*/tmp.*\)", - ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + ' running command "git archive -o .*/target/test.tar --prefix=testrepository/ HEAD"', + " \(in testrepository\)", + ' running command "gzip -nf .*/target/test.tar"', " \(in .*/tmp.*\)", ]) run_check() @@ -1758,7 +1760,14 @@ def run_check(): expected = '\n'.join([ ' running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"', " \(in .*/tmp.*\)", - ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + ' running command "git archive -o .*/target/test.tar --prefix=testrepository/ HEAD"', + " \(in testrepository\)", + ' running command "git submodule foreach ' + + '\'git archive -o \$name.tar --prefix=testrepository/\$path/ HEAD\'"', + " \(in testrepository\)", + ' running command "git submodule foreach \'tar -f .*/target/test.tar --concatenate \$name.tar\'"', + " \(in testrepository\)", + ' running command "gzip -nf .*/target/test.tar"', " \(in .*/tmp.*\)", ]) run_check() @@ -1770,7 +1779,14 @@ def run_check(): " \(in .*/tmp.*\)", ' running command "git checkout 8456f86 && git submodule update"', " \(in testrepository\)", - ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + ' running command "git archive -o .*/target/test.tar --prefix=testrepository/ HEAD"', + " \(in testrepository\)", + ' running command "git submodule foreach ' + + '\'git archive -o \$name.tar --prefix=testrepository/\$path/ HEAD\'"', + " \(in testrepository\)", + ' running command "git submodule foreach \'tar -f .*/target/test.tar --concatenate \$name.tar\'"', + " \(in testrepository\)", + ' running command "gzip -nf .*/target/test.tar"', " \(in .*/tmp.*\)", ]) run_check() @@ -1781,7 +1797,9 @@ def run_check(): " \(in .*/tmp.*\)", ' running command "git checkout 8456f86"', " \(in testrepository\)", - ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + ' running command "git archive -o .*/target/test.tar --prefix=testrepository/ HEAD"', + " \(in testrepository\)", + ' running command "gzip -nf .*/target/test.tar"', " \(in .*/tmp.*\)", ]) run_check()