Refactor logging and bioio. (#3351)

* Refactor logging. * Refactor addOptions. * Refactor. * Refactor. * Refactor. * Add lower. * Add lower. * Account parse stats arg correctly. * Add back Cactus-dependent naming. * Cruft. * Legacy log levels. * Updates. * Delete options.py * Updates. * Copyright. * Copyright. * Don't use __all__. * Update setup. * Replace deprecated imp library. * Run make sort_imports. * Run make format. * Switch back to imp. * Setup update. * Setup update. * Make sure sorted imports use VERTICAL. * Clean up launch cluster. * Refactor launch cluster. * Run black after sorting imports. * Refactor spot worker bid. * Fix test that is suddenly now running. * Better message. * SLURM Error? * Import error. * Fix import. * Resolve error and add deprecation warnings. * Add skipped doc tests. * Update. * Imports. * Suppress extraneous logging while testing. * Cruft. * Set logging. Remove docker in docker test. * Check upgrade to latest dev dependencies. * Update pytest version. * Update pytest version. * Update. * Suppress logging with env var. * Dev reqs versioning. * Add integration test. * Cruft. * Fix path. * Clean up utils test.
DataBiosphere · Dec 16, 2020 · 98dbf33 · 98dbf33
1 parent 55791fe
commit 98dbf33
Show file tree

Hide file tree

Showing 150 changed files with 1,565 additions and 1,867 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,11 +1,14 @@
 image: quay.io/vgteam/vg_ci_prebake:latest
 # Note that we must run in a privileged container for our internal Docker daemon to come up.
 
+variables:
+  PYTHONIOENCODING: "utf-8"
+  DEBIAN_FRONTEND: "noninteractive"
+
 before_script:
   - startdocker || true
   - docker info
   - cat /etc/hosts
-  - export PYTHONIOENCODING=utf-8
   - mkdir -p ~/.kube && cp "$GITLAB_SECRET_FILE_KUBE_CONFIG" ~/.kube/config
   - mkdir -p ~/.aws && cp "$GITLAB_SECRET_FILE_AWS_CREDENTIALS" ~/.aws/credentials
 
@@ -37,7 +40,7 @@ py36_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq
+    - apt update && apt install -y tzdata jq
     - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && pip install pycparser && make develop extras=[all] && pip install htcondor awscli==1.16.272
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
@@ -49,7 +52,7 @@ py37_batch_systems:
   stage: main_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
+    - apt update && apt install -y tzdata jq python3.7 python3.7-dev
     - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli==1.16.272
     - make test tests=src/toil/test/batchSystems/batchSystemTest.py
     - make test tests=src/toil/test/mesos/MesosDataStructuresTest.py
@@ -58,17 +61,18 @@ py37_cwl_v1.0:
   stage: main_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
+    - apt update && apt install -y tzdata jq python3.7 python3.7-dev
     - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[cwl,aws]
     - mypy --ignore-missing-imports --no-strict-optional $(pwd)/src/toil/cwl/cwltoil.py  # make this a separate linting stage
     - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
     - make test tests=src/toil/test/cwl/cwlTest.py::CWLv10Test
+    - make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testCwlexample
 
 py37_cwl_v1.1:
   stage: main_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
+    - apt update && apt install -y tzdata jq python3.7 python3.7-dev
     - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[cwl,aws]
     - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
     - make test tests=src/toil/test/cwl/cwlTest.py::CWLv11Test
@@ -77,7 +81,7 @@ py37_cwl_v1.2:
   stage: main_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
+    - apt update && apt install -y tzdata jq python3.7 python3.7-dev
     - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[cwl,aws]
     - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
     - make test tests=src/toil/test/cwl/cwlTest.py::CWLv12Test
@@ -86,7 +90,7 @@ py37_wdl:
   stage: main_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev default-jre
+    - apt update && apt install -y tzdata jq python3.7 python3.7-dev default-jre
     - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all]
     - which java &> /dev/null || { echo >&2 "Java is not installed.  Install java to run these tests."; exit 1; }
     - make test tests=src/toil/test/wdl/toilwdlTest.py  # needs java (default-jre) to run "GATK.jar"
@@ -96,7 +100,7 @@ py37_jobstore_and_provisioning:
   stage: main_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
+    - apt update && apt install -y tzdata jq python3.7 python3.7-dev
     - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor
     - make test tests=src/toil/test/jobStores/jobStoreTest.py
     - make test tests=src/toil/test/sort/sortTest.py
@@ -110,16 +114,17 @@ py37_main:
   stage: basic_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
+    - apt update && apt install -y tzdata jq python3.7 python3.7-dev
     - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor
     - make test tests=src/toil/test/src
     - make test tests=src/toil/test/utils
+#    - make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testDocker
 
 py37_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
+    - apt update && apt install -y tzdata jq python3.7 python3.7-dev
     - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && pip install pycparser && make develop extras=[all] && pip install htcondor awscli==1.16.272
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
@@ -129,7 +134,7 @@ py37_integration:
   stage: integration
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
+    - apt update && apt install -y tzdata jq python3.7 python3.7-dev
     - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli==1.16.272
     - export TOIL_TEST_INTEGRATIVE=True
     - export TOIL_AWS_KEYNAME=id_rsa
@@ -143,7 +148,7 @@ py37_provisioner_integration:
   stage: integration
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata awscli jq python3.7 python3.7-dev
+    - apt update && apt install -y tzdata awscli jq python3.7 python3.7-dev
     - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli==1.16.272
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
@@ -155,6 +160,7 @@ py37_provisioner_integration:
     - make push_docker
     - make test tests=src/toil/test/sort/sortTest.py
     - make test tests=src/toil/test/provisioners/clusterScalerTest.py
+    - make test tests=src/toil/test/utils/utilsTest.py::UtilsTest::testAWSProvisionerUtils
     # - python -m pytest --duration=0 -s -r s src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSRestartTest::testAutoScaledCluster
     # - python -m pytest -s src/toil/test/provisioners/aws/awsProvisionerTest.py
     # - python -m pytest -s src/toil/test/provisioners/gceProvisionerTest.py  # needs env vars set to run
@@ -164,7 +170,7 @@ py38_main:
   stage: basic_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.8 python3.8-dev
+    - apt update && apt install -y tzdata jq python3.8 python3.8-dev
     - virtualenv -p python3.8 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor
     - make test tests=src/toil/test/src
     - make test tests=src/toil/test/utils
@@ -173,7 +179,7 @@ py38_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.8 python3.8-dev
+    - apt update && apt install -y tzdata jq python3.8 python3.8-dev
     - virtualenv -p python3.8 venv && . venv/bin/activate && make prepare && pip install pycparser && make develop extras=[all] && pip install htcondor awscli==1.16.272
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
@@ -184,7 +190,7 @@ py37_cactus_integration:
   stage: integration
   script:
     - set -e
-    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata awscli jq python3.7 python3.7-dev
+    - apt update && apt install -y tzdata awscli jq python3.7 python3.7-dev
     - virtualenv --system-site-packages --python python3.7 venv
     - . venv/bin/activate
     - pip install .[aws,kubernetes]

diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 UCSC Computational Genomics Lab
+# Copyright (C) 2015-2021 Regents of the University of California
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -113,7 +113,7 @@ clean_sdist:
 # We always claim to be Travis, so that local test runs will not skip Travis tests.
 test: check_venv check_build_reqs
 	TRAVIS=true \
-	    python -m pytest --cov=toil --duration=0 -s -r s $(tests)
+	    python -m pytest --cov=toil --durations=0 -s -r s $(tests)
 
 
 # This target will skip building docker and all docker based tests
@@ -202,9 +202,7 @@ check_build_reqs:
 		|| ( printf "$(red)Build requirements are missing. Run 'make prepare' to install them.$(normal)\n" ; false )
 
 prepare: check_venv
-	pip install mock==1.0.1 pytest==4.3.1 pytest-cov==2.6.1 stubserver==1.0.1 \
-		pytest-timeout==1.3.3 setuptools==45.3.0 'sphinx>=2.4.4,<3' \
-		cwltest mypy flake8 flake8-bugbear black isort pydocstyle
+	pip install -r requirements-dev.txt
 
 check_venv:
 	@python -c 'import sys, os; sys.exit( int( 0 if "VIRTUAL_ENV" in os.environ else 1 ) )' \
@@ -231,10 +229,12 @@ PYSOURCES=$(shell find src -name '*.py') setup.py version_template.py
 # Linting and code style related targets
 ## sorting imports using isort: https://github.com/timothycrosley/isort
 sort_imports: $(PYSOURCES)
-	isort $^
+	isort -m VERTICAL $^
+	make format
 
 remove_unused_imports: $(PYSOURCES)
 	autoflake --in-place --remove-all-unused-imports $^
+	make format
 
 format: $(wildcard src/toil/cwl/*.py)
 	black $^

diff --git a/docker/Dockerfile.py b/docker/Dockerfile.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 Regents of the University of California
+# Copyright (C) 2015-2020 Regents of the University of California
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -0,0 +1,16 @@
+mock==4.0.3
+pytest==6.2.0
+pytest-cov==2.10.1
+pytest-timeout==1.4.2
+stubserver==1.1
+setuptools==51.0.0
+sphinx==3.3.1
+cwltest==2.0.20200626112502
+mypy==0.790
+flake8==3.8.4
+flake8-bugbear==20.11.1
+black
+isort
+pydocstyle
+autoflake
+isort
diff --git a/setup.py b/setup.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 Regents of the University of California
+# Copyright (C) 2015-2021 Regents of the University of California
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,13 +11,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import imp
+import os
+
+from tempfile import NamedTemporaryFile
 from setuptools import find_packages, setup
 
 
-def runSetup():
+def run_setup():
     """
     Calls setup(). This function exists so the setup() invocation preceded more internal
-    functionality. The `version` module is imported dynamically by importVersion() below.
+    functionality. The `version` module is imported dynamically by import_version() below.
     """
     boto = 'boto==2.48.0'
     boto3 = 'boto3>=1.7.50, <2.0'
@@ -51,7 +55,6 @@ def runSetup():
         addict,
         pytz,
         enlighten]
-
     aws_reqs = [
         boto,
         boto3,
@@ -75,7 +78,6 @@ def runSetup():
         pymesos,
         psutil]
     wdl_reqs = []
-
 
     # htcondor is not supported by apple
     # this is tricky to conditionally support in 'all' due
@@ -89,7 +91,6 @@ def runSetup():
         kubernetes_reqs + \
         mesos_reqs
 
-
     setup(
         name='toil',
         version=version.distVersion,
@@ -135,7 +136,7 @@ def runSetup():
                                # Note that we intentionally include the top-level `test` package for
                                # functionality like the @experimental and @integrative decorators:
                                exclude=['*.test.*']),
-        package_data = {
+        package_data={
             '': ['*.yml', 'cloud-config'],
         },
         # Unfortunately, the names of the entry points are hard-coded elsewhere in the code base so
@@ -152,44 +153,27 @@ def runSetup():
                 '_toil_kubernetes_executor = toil.batchSystems.kubernetes:executor [kubernetes]']})
 
 
-def importVersion():
-    """
-    Load and return the module object for src/toil/version.py, generating it from the template if
-    required.
-    """
-    import imp
-    try:
-        # Attempt to load the template first. It only exists in a working copy cloned via git.
-        import version_template
-    except ImportError:
-        # If loading the template fails we must be in a unpacked source distribution and
-        # src/toil/version.py will already exist.
-        pass
-    else:
+def import_version():
+    """Return the module object for src/toil/version.py, generate from the template if required."""
+    if not os.path.exists('src/toil/version.py'):
         # Use the template to generate src/toil/version.py
-        import errno
-        import os
-        from tempfile import NamedTemporaryFile
-
-        new = version_template.expand_()
-        try:
-            with open('src/toil/version.py') as f:
-                old = f.read()
-        except IOError as e:
-            if e.errno == errno.ENOENT:
-                old = None
-            else:
-                raise
+        import version_template
+        with NamedTemporaryFile(mode='w', dir='src/toil', prefix='version.py.', delete=False) as f:
+            f.write(version_template.expand_())
+        os.rename(f.name, 'src/toil/version.py')
 
-        if old != new:
-            with NamedTemporaryFile(mode='w', dir='src/toil', prefix='version.py.', delete=False) as f:
-                f.write(new)
-            os.rename(f.name, 'src/toil/version.py')
     # Unfortunately, we can't use a straight import here because that would also load the stuff
-    # defined in src/toil/__init__.py which imports modules from external dependencies that may
+    # defined in "src/toil/__init__.py" which imports modules from external dependencies that may
     # yet to be installed when setup.py is invoked.
+    #
+    # This is also the reason we cannot switch from the "deprecated" imp library
+    # and use:
+    #     from importlib.machinery import SourceFileLoader
+    #     return SourceFileLoader('toil.version', path='src/toil/version.py').load_module()
+    #
+    # Because SourceFileLoader will error and load "src/toil/__init__.py" .
     return imp.load_source('toil.version', 'src/toil/version.py')
 
 
-version = importVersion()
-runSetup()
+version = import_version()
+run_setup()
diff --git a/setup_gitlab_docker.py b/setup_gitlab_docker.py
@@ -1,3 +1,16 @@
+# Copyright (C) 2015-2021 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 import json
 import os
 import subprocess

diff --git a/setup_gitlab_ssh.py b/setup_gitlab_ssh.py
@@ -1,3 +1,16 @@
+# Copyright (C) 2015-2021 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 import json
 import os