Merge pull request #15085 from bernt-matthias/topic/decompress-assert…

…ions Decompress history data for testing assertions
galaxyproject · Jun 21, 2023 · a68a453 · a68a453
2 parents 8c4598f + fcb9809
commit a68a453
Show file tree

Hide file tree

Showing 5 changed files with 1,013 additions and 706 deletions.
diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py
@@ -73,7 +73,7 @@ def _retrieve_file_content(filename: str) -> bytes:
     assertions = attributes.get("assert_list", None)
     if attributes is not None and assertions is not None:
         try:
-            verify_assertions(output_content, attributes["assert_list"])
+            verify_assertions(output_content, attributes["assert_list"], attributes.get("decompress"))
         except AssertionError as err:
             errmsg = f"{item_label} different than expected\n"
             errmsg += unicodify(err)

diff --git a/lib/galaxy/tool_util/verify/asserts/__init__.py b/lib/galaxy/tool_util/verify/asserts/__init__.py
@@ -4,8 +4,10 @@
     getfullargspec,
     getmembers,
 )
+from tempfile import NamedTemporaryFile
 
 from galaxy.util import unicodify
+from galaxy.util.compression_utils import get_fileobj
 
 log = logging.getLogger(__name__)
 
@@ -30,9 +32,15 @@
             assertion_functions[member] = value
 
 
-def verify_assertions(data: bytes, assertion_description_list):
+def verify_assertions(data: bytes, assertion_description_list, decompress=None):
     """This function takes a list of assertions and a string to check
     these assertions against."""
+    if decompress:
+        with NamedTemporaryFile() as tmpfh:
+            tmpfh.write(data)
+            tmpfh.flush()
+            with get_fileobj(tmpfh.name, mode="rb", compressed_formats=None) as fh:
+                data = fh.read()
     for assertion_description in assertion_description_list:
         verify_assertion(data, assertion_description)
 

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -1720,7 +1720,15 @@ it may be inconvenient to upload the entiry file and this can be used instead.
     <xs:attribute name="decompress" type="PermissiveBoolean">
       <xs:annotation>
         <xs:documentation xml:lang="en"><![CDATA[
-When this attribute is true and ``compare`` is set to ``diff``, try to decompress files if needed. This flag is useful for testing compressed outputs that are non-deterministic despite having deterministic decompressed contents. By default, only files compressed with bz2, gzip and zip will be automatically decompressed. This is available in Galaxy since release 17.05 and was introduced in [pull request #3550](https://github.com/galaxyproject/galaxy/pull/3550).
+If this attribute is true then try to decompress files if needed. This applies to
+test assertions expressed with ``assert_contents`` or ``compare`` set to anything
+but ``sim_size``. 
+This flag is useful for testing compressed outputs that are non-deterministic
+despite having deterministic decompressed contents. By default, only files compressed
+with bz2, gzip and zip will be automatically decompressed.
+Note, for specifying assertions for compressed as well as decompressed output
+the corresponding output tag can be specified multiple times.
+This is available in Galaxy since release 17.05 and was introduced in [pull request #3550](https://github.com/galaxyproject/galaxy/pull/3550).
 ]]></xs:documentation>
       </xs:annotation>
     </xs:attribute>

diff --git a/test/functional/tools/validation_tar_gz.xml b/test/functional/tools/validation_tar_gz.xml
@@ -1,4 +1,7 @@
 <tool id="validation_tar_gz" name="Validation (tar.gz)" version="1.0">
+    <!-- tool tests for assertions on gzipped outputs 
+         also tests that test outputs can be given multiple times 
+         (eg to specify assertions on compressed and upcompressed data)-->
     <command><![CDATA[
 cat '$input' > '$output'
     ]]></command>
@@ -9,10 +12,49 @@ cat '$input' > '$output'
         <data name="output" />
     </outputs>
     <tests>
+        <test>
+            <param name="input" value="1.fastqsanger.gz" />
+            <output name="output">
+                <assert_contents>
+                    <has_size value="161"/>
+                    <has_line line="@1831_573_1004/1" negate="true"/>
+                </assert_contents>
+            </output>
+            <output name="output" decompress="true">
+                <assert_contents>
+                    <has_size value="177"/>
+                    <has_line line="@1831_573_1004/1"/>
+                </assert_contents>
+            </output>
+        </test>
+        <!-- assure that the assertions of the 2nd use of the output named "output" are considered -->
+        <test expect_test_failure="true">
+            <param name="input" value="1.fastqsanger.gz" />
+            <output name="output">
+                <assert_contents>
+                    <has_size value="161"/>
+                    <has_line line="@1831_573_1004/1" negate="true"/>
+                </assert_contents>
+            </output>
+            <output name="output" decompress="true">
+                <assert_contents>
+                    <has_size value="1M"/>
+                    <has_line line="I'm not here"/>
+                </assert_contents>
+            </output>
+        </test>
         <test>
             <param name="input" value="testdir1.tar.gz" />
+            <!-- decompression works only for .gz but not .tar.gz
+                 hence has_size is the same as for decompress="false"-->
+            <output name="output" decompress="true">
+                <assert_contents>
+                    <has_size value="10240"/>
+                </assert_contents>
+            </output>
             <output name="output">
                 <assert_contents>
+                    <has_size value="10240"/>
                     <!-- Test 0: Empty tag, checks for presence of member -->
                     <has_archive_member path="testdir1/file1"></has_archive_member>