Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test wildcards #98

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Test wildcards #98

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 49 additions & 7 deletions tests/test_extract.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ class TestExtract(TestZstash):
"""
# `zstash extract` is tested in TestExtract and TestExtractParallel.
# x = on, no mark = off, b = both on and off tested
# option | ExtractVerbose | ExtractKeep | ExtractCache | ExtractParallel |
# --hpss |x|x|x|x|
# --workers | | | |x|
# --cache | | |x| |
# --keep | |x| | |
# -v |x| | |b|
# option | ExtractVerbose | ExtractKeep | ExtractCache | ExtractWildcard | ExtractParallel |
# --hpss |x|x|x|x|x|
# --workers | | | | |x|
# --cache | | |x| | |
# --keep | |x| | | |
# -v |x| | | |b|

def helperExtractVerbose(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
"""
Expand Down Expand Up @@ -98,7 +98,7 @@ def helperExtractKeep(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
if not compare(os.listdir(self.cache),
['index.db', '000000.tar', '000001.tar', '000002.tar', '000003.tar', '000004.tar']):
error_message = 'The zstash directory does not contain expected files.\nIt has: {}'.format(
os.listdir(d))
os.listdir(self.cache))
self.stop(error_message)
os.chdir(TOP_LEVEL)
expected_present = ['Extracting file0.txt', 'Extracting file0_hard.txt', 'Extracting file0_soft.txt',
Expand Down Expand Up @@ -131,6 +131,45 @@ def helperExtractCache(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
error_message = 'The zstash cache does not contain expected files.\nIt has: {}'.format(files)
self.stop(error_message)

def helperExtractWildcard(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
"""
Test `zstash extract` with wildcard.
"""
self.assertWorkspace()
print_in_box(test_name)
self.hpss_path = hpss_path
if self.hpss_path.lower() == 'none':
use_hpss = False
else:
use_hpss = True
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
write_file('aa0.txt', 'aa0')
write_file('aa1.txt', 'aa1')
write_file('ab2.txt', 'ab2')
write_file('ab3.txt', 'ab3')
os.chdir(TOP_LEVEL)
self.create(use_hpss, zstash_path)
# Rename test_dir to backup_dir so that the new test_dir won't have the files we're trying to extract.
os.rename(self.test_dir, self.backup_dir)
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
if not use_hpss:
shutil.copytree('{}/{}/{}'.format(TOP_LEVEL, self.backup_dir, self.cache), self.copy_dir)
cmd = '{}zstash extract --hpss={} ab*'.format(zstash_path, self.hpss_path)
output, err = run_cmd(cmd)
print(output + err)
expected_present = ['Extracting ab2.txt', 'Extracting ab3.txt']
self.check_strings(cmd, output + err, expected_present, [])
cmd = '{}zstash extract --hpss={} a*'.format(zstash_path, self.hpss_path)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interestingly, adding double quotes seems to cause a failure and not adding them allows the tests to pass -- the reverse of our experience as users. I believe this is because subprocess.Popen requires cmd to be converted to a list and I imagine the conversion encapsulates a* in quotes. Not sure if there's a way to automate a test in such a way as to use the quotes.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Try using shell=True? If that doesn't work, perhaps we should just close #96 without adding automated testing.

output, err = run_cmd(cmd)
print(output + err)
expected_present = ['Extracting aa0.txt', 'Extracting aa1.txt',
'Not extracting ab2.txt', 'Not extracting ab3.txt']
self.check_strings(cmd, output + err, expected_present, [])
os.chdir(TOP_LEVEL)


def testExtractVerbose(self):
self.helperExtractVerbose('testExtractVerbose', 'none')

Expand All @@ -148,6 +187,9 @@ def testExtractKeepHPSS(self):
def testExtractCache(self):
self.helperExtractCache('testExtractCache', 'none')

def testExtractWildcard(self):
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need a test with HPSS too?

self.helperExtractWildcard('testExtractWildcard', 'none')

def testExtractCacheHPSS(self):
self.conditional_hpss_skip()
self.helperExtractCache('testExtractCacheHPSS', HPSS_ARCHIVE)
Expand Down
12 changes: 6 additions & 6 deletions tests/test_extract_parallel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ class TestExtractParallel(TestZstash):
"""
# `zstash extract` is tested in TestExtract and TestExtractParallel.
# x = on, no mark = off, b = both on and off tested
# option | ExtractVerbose | Extract | ExtractCache | ExtractParallel |
# --hpss |x|x|x|x|
# --workers | | | |x|
# --cache | | |x| |
# --keep | |x| | |
# -v |x| | |b|
# option | ExtractVerbose | Extract | ExtractCache | ExtractWildcard | ExtractParallel |
# --hpss |x|x|x|x|x|
# --workers | | | | |x|
# --cache | | |x| | |
# --keep | |x| | | |
# -v |x| | | |b|

def helperExtractParallel(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
"""
Expand Down