diff --git a/tests_integration/__init__.py b/tests_integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests2/test_cache_fs.py b/tests_integration/test_cache_fs.py similarity index 81% rename from tests2/test_cache_fs.py rename to tests_integration/test_cache_fs.py index 209496b9..a315c711 100644 --- a/tests2/test_cache_fs.py +++ b/tests_integration/test_cache_fs.py @@ -1,7 +1,7 @@ import os import unittest -from tests2.base import TOP_LEVEL, TestZstash, run_cmd +from tests_integration.utils import TestZstash, run_cmd class TestCacheFs(TestZstash): @@ -17,7 +17,7 @@ def test_hpss_none_fs_on(self): # internal symlink (in same dir, in different dir), external symlink # internal hard link (in same dir, in different dir), external hard link, broken hard link self.setup_dirs(include_broken_symlink=False) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/") + os.chdir(f"{self.work_dir}/zstash_src/") # Test zstash_src before create self.assertTrue(os.path.islink("file0_soft.txt")) @@ -28,10 +28,10 @@ def test_hpss_none_fs_on(self): self.assertFalse(os.path.islink("file_not_included_hard.txt")) self.assertFalse(os.path.islink("original_was_deleted_hard.txt")) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/") - cmd = f"zstash create --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache --follow-symlinks zstash_src" + os.chdir(f"{self.work_dir}/") + cmd = f"zstash create --hpss=none --cache={self.work_dir}/test_cache --follow-symlinks zstash_src" run_cmd(cmd) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/") + os.chdir(f"{self.work_dir}/zstash_src/") # Test zstash_src after create # Running `create` should not alter the source directory. @@ -44,9 +44,7 @@ def test_hpss_none_fs_on(self): self.assertFalse(os.path.islink("original_was_deleted_hard.txt")) os.chdir("../zstash_extracted") - cmd = ( - f"zstash extract --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache" - ) + cmd = f"zstash extract --hpss=none --cache={self.work_dir}/test_cache" run_cmd(cmd) # Test extraction from zstash_archive @@ -81,24 +79,22 @@ def test_hpss_none_fs_on_broken_symlink(self): # Cases: # broken symlink self.setup_dirs(include_broken_symlink=True) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/") + os.chdir(f"{self.work_dir}/zstash_src/") # Test zstash_src before create self.assertTrue(os.path.islink("original_was_deleted_soft.txt")) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/") - cmd = f"zstash create --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache --follow-symlinks zstash_src" + os.chdir(f"{self.work_dir}/") + cmd = f"zstash create --hpss=none --cache={self.work_dir}/test_cache --follow-symlinks zstash_src" run_cmd(cmd) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/") + os.chdir(f"{self.work_dir}/zstash_src/") # Test zstash_src after create # Running `create` should not alter the source directory. self.assertTrue(os.path.islink("original_was_deleted_soft.txt")) os.chdir("../zstash_extracted") - cmd = ( - f"zstash extract --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache" - ) + cmd = f"zstash extract --hpss=none --cache={self.work_dir}/test_cache" _, err = run_cmd(cmd) # This is ultimately caused by: # `Exception: Archive creation failed due to broken symlink.` @@ -111,7 +107,7 @@ def test_hpss_none_fs_on_broken_symlink(self): def test_hpss_none_fs_off(self): # self.setup_dirs(include_broken_symlink=False) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/") + os.chdir(f"{self.work_dir}/zstash_src/") # Test zstash_src before create self.assertTrue(os.path.islink("file0_soft.txt")) @@ -122,10 +118,10 @@ def test_hpss_none_fs_off(self): self.assertFalse(os.path.islink("file_not_included_hard.txt")) self.assertFalse(os.path.islink("original_was_deleted_hard.txt")) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/") - cmd = f"zstash create --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache zstash_src" + os.chdir(f"{self.work_dir}/") + cmd = f"zstash create --hpss=none --cache={self.work_dir}/test_cache zstash_src" run_cmd(cmd) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/") + os.chdir(f"{self.work_dir}/zstash_src/") # Test zstash_src after create # Running `create` should not alter the source directory. @@ -138,9 +134,7 @@ def test_hpss_none_fs_off(self): self.assertFalse(os.path.islink("original_was_deleted_hard.txt")) os.chdir("../zstash_extracted") - cmd = ( - f"zstash extract --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache" - ) + cmd = f"zstash extract --hpss=none --cache={self.work_dir}/test_cache" run_cmd(cmd) # Test extraction from zstash_archive @@ -172,24 +166,22 @@ def test_hpss_none_fs_off(self): def test_hpss_none_fs_off_broken_symlink(self): self.setup_dirs(include_broken_symlink=True) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/") + os.chdir(f"{self.work_dir}/zstash_src/") # Test zstash_src before create self.assertTrue(os.path.islink("original_was_deleted_soft.txt")) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/") - cmd = f"zstash create --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache zstash_src" + os.chdir(f"{self.work_dir}/") + cmd = f"zstash create --hpss=none --cache={self.work_dir}/test_cache zstash_src" run_cmd(cmd) - os.chdir(f"{TOP_LEVEL}/{self.work_dir}/zstash_src/") + os.chdir(f"{self.work_dir}/zstash_src/") # Test zstash_src after create # Running `create` should not alter the source directory. self.assertTrue(os.path.islink("original_was_deleted_soft.txt")) os.chdir("../zstash_extracted") - cmd = ( - f"zstash extract --hpss=none --cache={TOP_LEVEL}/{self.work_dir}/test_cache" - ) + cmd = f"zstash extract --hpss=none --cache={self.work_dir}/test_cache" run_cmd(cmd) # With fs off, this command completes successfully. diff --git a/tests_integration/test_tutorial_2020.py b/tests_integration/test_tutorial_2020.py new file mode 100644 index 00000000..6f782294 --- /dev/null +++ b/tests_integration/test_tutorial_2020.py @@ -0,0 +1,10 @@ +# Test 2020 tutorial workflow. +# https://github.com/E3SM-Project/zstash/pull/79/files + +from tests_integration.utils import TestZstash + + +class TestTutorial2020(TestZstash): + + def test_tutorial_2020(self): + pass diff --git a/tests_integration/test_tutorial_2024.py b/tests_integration/test_tutorial_2024.py new file mode 100644 index 00000000..251508ab --- /dev/null +++ b/tests_integration/test_tutorial_2024.py @@ -0,0 +1,111 @@ +# Test workflow similar to the 2024 tutorial. +# https://github.com/E3SM-Project/zstash/blob/add-tutorial-materials/tutorial_materials/zstash_demo.md + + +import os + +from tests_integration.utils import TestZstash, run_cmd + + +class TestTutorial2024(TestZstash): + + def test_tutorial_2024(self): + # This test can only be run on NERSC, using NERSC HPSS. + self.conditional_hpss_skip() + self.setup_dirs() + os.chdir(f"{self.work_dir}") + + files_to_include = "dir1/*.txt" # Should match file1.txt + cmd = f"zstash create --hpss={self.hpss_dir} --cache={self.cache_dir} --include={files_to_include} {self.dir_to_archive}" + output, err = run_cmd(cmd) + expected_present = [ + "Creating new tar archive", + "Archiving dir1/file1.txt", + "Transferring file to HPSS", + "Completed archive file", + ] + expected_absent = [ + "ERROR", + "Archiving file0.txt", + "Archiving file_empty.txt", + "Archiving file0_soft.txt", + "Archiving dir2/file1_soft.txt", + "Archiving file_not_included_soft.txt", + "Archiving file0_hard.txt", + "Archiving dir2/file1_hard.txt", + "Archiving file_not_included_hard.txt", + "Archiving original_was_deleted_hard.txt", + ] + self.check_strings(cmd, output + err, expected_present, expected_absent) + + os.mkdir("check_output") + os.chdir("check_output") + cmd = f"zstash check --hpss={self.hpss_dir}" + output, err = run_cmd(cmd) + expected_present = [ + "Transferring file from HPSS", + "Opening tar archive", + "Checking dir1/file1.txt", + "No failures detected when checking the files", + ] + expected_absent = [ + "ERROR", + "Checking file0.txt", + "Checking file_empty.txt", + "Checking file0_soft.txt", + "Checking dir2/file1_soft.txt", + "Checking file_not_included_soft.txt", + "Checking file0_hard.txt", + "Checking dir2/file1_hard.txt", + "Checking file_not_included_hard.txt", + "Checking original_was_deleted_hard.txt", + ] + self.check_strings(cmd, output + err, expected_present, expected_absent) + os.chdir(f"{self.work_dir}") + + os.mkdir("ls_output") + os.chdir("ls_output") + cmd = f"zstash ls --hpss={self.hpss_dir}" + output, err = run_cmd(cmd) + expected_present = [ + "dir1/file1.txt", + ] + expected_absent = [ + "ERROR", + "file0.txt", + "file_empty.txt", + "file0_soft.txt", + "dir2/file1_soft.txt", + "file_not_included_soft.txt", + "file0_hard.txt", + "dir2/file1_hard.txt", + "file_not_included_hard.txt", + "original_was_deleted_hard.txt", + ] + self.check_strings(cmd, output + err, expected_present, expected_absent) + os.chdir(f"{self.work_dir}") + + os.mkdir("extract_output") + os.chdir("extract_output") + cmd = f"zstash extract --hpss={self.hpss_dir}" + output, err = run_cmd(cmd) + expected_present = [ + "Transferring file from HPSS", + "Opening tar archive", + "Extracting dir1/file1.txt", + "No failures detected when extracting the files", + ] + expected_absent = [ + "ERROR", + "Extracting file0.txt", + "Extracting file_empty.txt", + "Extracting file0_soft.txt", + "Extracting dir2/file1_soft.txt", + "Extracting file_not_included_soft.txt", + "Extracting file0_hard.txt", + "Extracting dir2/file1_hard.txt", + "Extracting file_not_included_hard.txt", + "Extracting original_was_deleted_hard.txt", + ] + self.check_strings(cmd, output + err, expected_present, expected_absent) + os.chdir(f"{self.work_dir}") diff --git a/tests2/base.py b/tests_integration/utils.py similarity index 65% rename from tests2/base.py rename to tests_integration/utils.py index 644f3577..78eb6e31 100644 --- a/tests2/base.py +++ b/tests_integration/utils.py @@ -1,10 +1,8 @@ """ -Run the test suite with `python -m unittest tests2/test_*.py` +Run the test suite with `python -m unittest tests_integration/test_*.py` -tests2/ is a successor testing directory to tests/ -All new tests should be written in tests2/ tests/ groups testing by zstash command (e.g., `create`, `extract`) -tests2/ groups testing by more logical workflows that test multiple zstash commands. +tests_integration/ groups testing by more logical workflows that test multiple zstash commands. The goal of tests2/ is to be able to follow the commands as if you were just reading a bash script. """ @@ -16,6 +14,8 @@ import unittest from typing import List, Tuple +# Still using unittest instead of pytest, because we need the setup and teardown methods applying to all tests across all test files. + # https://bugs.python.org/issue43743 # error: Module has no attribute "_USE_CP_SENDFILE" shutil._USE_CP_SENDFILE = False # type: ignore @@ -25,6 +25,10 @@ # This is used to ensure we are changing into the correct subdirectories and parent directories. TOP_LEVEL = os.getcwd() +# TODO: add an equivalent SKIP_GLOBUS +# Skip all HPSS tests. Decreases test runtime. +SKIP_HPSS = False + def create_directories(dir_names: List[str]): for dir in dir_names: @@ -81,6 +85,7 @@ def print_in_box(string): print("*" * 40) +############################################################################### class TestZstash(unittest.TestCase): """ Base test class. @@ -91,10 +96,19 @@ def setUp(self): Set up a test. This is run before every test method. """ os.chdir(TOP_LEVEL) - # The directory we'll be working in. - self.work_dir = "zstash_work_dir" - # The HPSS path - self.hpss_path = None + # Directories + work_dir = "zstash_work_dir" + + self.work_dir = f"{TOP_LEVEL}/{work_dir}/" + self.cache_dir = f"{TOP_LEVEL}/{work_dir}/zstash_cache_dir/" + self.dir_to_archive = f"{TOP_LEVEL}/{work_dir}/zstash_src/" + + self.relative_work_dir = f"{work_dir}/" + self.relative_cache_dir = "zstash_cache_dir/" + self.relative_dir_to_archive = "zstash_src/" + + self.hpss_dir = "zstash_hpss_dir" + # The mtime to compare back to, to make sure we're not modifying the source directory. self.mtime_start = None @@ -106,20 +120,25 @@ def tearDown(self): """ os.chdir(TOP_LEVEL) print("Removing test files, both locally and at the HPSS repo") - for d in [self.work_dir]: + for d in [self.relative_work_dir]: if os.path.exists(d): shutil.rmtree(d) - if self.hpss_path and self.hpss_path.lower() != "none": - cmd = "hsi rm -R {}".format(self.hpss_path) + if self.hpss_dir and self.hpss_dir.lower() != "none": + cmd = "hsi rm -R {}".format(self.hpss_dir) run_cmd(cmd) + def conditional_hpss_skip(self): + skip_str = "Skipping HPSS tests." + if SKIP_HPSS: + self.skipTest("SKIP_HPSS is True. {}".format(skip_str)) + elif os.system("which hsi") != 0: + self.skipTest("This system does not have hsi. {}".format(skip_str)) + def assert_source_unchanged(self): """ Assert that the source directory has not been changed. """ - mtime_current = os.stat(f"{TOP_LEVEL}/{self.work_dir}/zstash_src")[ - stat.ST_MTIME - ] + mtime_current = os.stat(f"{self.work_dir}/zstash_src")[stat.ST_MTIME] if self.mtime_start != mtime_current: self.stop( f"Source directory was modified! {self.mtime_start} != {mtime_current}" @@ -166,32 +185,32 @@ def check_strings( print_in_box(error_message) self.stop(error_message) - def setup_dirs(self, include_broken_symlink=True): + def setup_dirs(self, include_broken_symlink=False): """ Set up directories for testing. """ create_directories( [ - self.work_dir, - f"{self.work_dir}/zstash_src/", - f"{self.work_dir}/zstash_src/empty_dir", - f"{self.work_dir}/zstash_src/dir1", - f"{self.work_dir}/zstash_src/dir2", - f"{self.work_dir}/zstash_not_src", - f"{self.work_dir}/zstash_extracted", + self.relative_work_dir, + f"{self.relative_work_dir}/zstash_src/", + f"{self.relative_work_dir}/zstash_src/empty_dir", + f"{self.relative_work_dir}/zstash_src/dir1", + f"{self.relative_work_dir}/zstash_src/dir2", + f"{self.relative_work_dir}/zstash_not_src", + f"{self.relative_work_dir}/zstash_extracted", ] ) write_files( [ - (f"{self.work_dir}/zstash_src/file0.txt", "file0 stuff"), - (f"{self.work_dir}/zstash_src/file_empty.txt", ""), - (f"{self.work_dir}/zstash_src/dir1/file1.txt", "file1 stuff"), + (f"{self.relative_work_dir}/zstash_src/file0.txt", "file0 stuff"), + (f"{self.relative_work_dir}/zstash_src/file_empty.txt", ""), + (f"{self.relative_work_dir}/zstash_src/dir1/file1.txt", "file1 stuff"), ( - f"{self.work_dir}/zstash_not_src/file_not_included.txt", + f"{self.relative_work_dir}/zstash_not_src/file_not_included.txt", "file_not_included stuff", ), ( - f"{self.work_dir}/zstash_not_src/this_will_be_deleted.txt", + f"{self.relative_work_dir}/zstash_not_src/this_will_be_deleted.txt", "deleted stuff", ), ] @@ -205,17 +224,17 @@ def setup_dirs(self, include_broken_symlink=True): # But os.symlink('dir/original_file`, 'dir/soft_link') will soft link dir/soft_link to dir/dir/original_file! # That is, the link's directory will always be used as the base path for the original file. # 1) Link to a file in the same subdirectory - ("file0.txt", f"{self.work_dir}/zstash_src/file0_soft.txt"), + ("file0.txt", f"{self.relative_work_dir}/zstash_src/file0_soft.txt"), # There is a way around this, though: use an absolute path. # 2) Link to a file in a different subdirectory ( - f"{TOP_LEVEL}/{self.work_dir}/zstash_src/dir1/file1.txt", - f"{self.work_dir}/zstash_src/dir2/file1_soft.txt", + f"{self.work_dir}/zstash_src/dir1/file1.txt", + f"{self.relative_work_dir}/zstash_src/dir2/file1_soft.txt", ), # 3) Link to a file outside the directory to be archived ( - f"{TOP_LEVEL}/{self.work_dir}/zstash_not_src/file_not_included.txt", - f"{self.work_dir}/zstash_src/file_not_included_soft.txt", + f"{self.work_dir}/zstash_not_src/file_not_included.txt", + f"{self.relative_work_dir}/zstash_src/file_not_included_soft.txt", ), ] ) @@ -224,34 +243,32 @@ def setup_dirs(self, include_broken_symlink=True): [ # Note that here, we do need to include the relative path for both. ( - f"{self.work_dir}/zstash_src/file0.txt", - f"{self.work_dir}/zstash_src/file0_hard.txt", + f"{self.relative_work_dir}/zstash_src/file0.txt", + f"{self.relative_work_dir}/zstash_src/file0_hard.txt", ), ( - f"{TOP_LEVEL}/{self.work_dir}/zstash_src/dir1/file1.txt", - f"{self.work_dir}/zstash_src/dir2/file1_hard.txt", + f"{self.work_dir}/zstash_src/dir1/file1.txt", + f"{self.relative_work_dir}/zstash_src/dir2/file1_hard.txt", ), ( - f"{TOP_LEVEL}/{self.work_dir}/zstash_not_src/file_not_included.txt", - f"{self.work_dir}/zstash_src/file_not_included_hard.txt", + f"{self.work_dir}/zstash_not_src/file_not_included.txt", + f"{self.relative_work_dir}/zstash_src/file_not_included_hard.txt", ), # Also include a broken hard link ( - f"{self.work_dir}/zstash_not_src/this_will_be_deleted.txt", - f"{self.work_dir}/zstash_src/original_was_deleted_hard.txt", + f"{self.relative_work_dir}/zstash_not_src/this_will_be_deleted.txt", + f"{self.relative_work_dir}/zstash_src/original_was_deleted_hard.txt", ), ], do_symlink=False, ) if include_broken_symlink: os.symlink( - f"{self.work_dir}/zstash_not_src/this_will_be_deleted.txt", - f"{self.work_dir}/zstash_src/original_was_deleted_soft.txt", + f"{self.relative_work_dir}/zstash_not_src/this_will_be_deleted.txt", + f"{self.relative_work_dir}/zstash_src/original_was_deleted_soft.txt", ) - os.remove(f"{self.work_dir}/zstash_not_src/this_will_be_deleted.txt") - self.mtime_start = os.stat(f"{TOP_LEVEL}/{self.work_dir}/zstash_src")[ - stat.ST_MTIME - ] + os.remove(f"{self.relative_work_dir}/zstash_not_src/this_will_be_deleted.txt") + self.mtime_start = os.stat(f"{self.work_dir}/zstash_src")[stat.ST_MTIME] if __name__ == "__main__":