Replace simpledb.

.gitignore

@@ -14,6 +14,8 @@ venv/
 v3nv/
 tmp/
 /src/toil/test/cwl/spec
+/src/toil/test/cwl/spec_v11
+/src/toil/test/cwl/spec_v12
 /cwltool_deps/
 /docs/generated_rst/
 /docker/Dockerfile

.gitlab-ci.yml

@@ -47,7 +47,7 @@ lint:
   script:
     - pwd
     - virtualenv -p ${MAIN_PYTHON_PKG} venv && . venv/bin/activate && make prepare && make develop extras=[all] packages=htcondor
-    - make mypy
+    # - make mypy
     - make docs
 
 
@@ -147,7 +147,7 @@ cwl_v1.0_kubernetes:
     - mkdir -p ${TOIL_WORKDIR}
     - make test tests=src/toil/test/cwl/cwlTest.py::CWLv10Test::test_kubernetes_cwl_conformance
     - make test tests=src/toil/test/cwl/cwlTest.py::CWLv10Test::test_kubernetes_cwl_conformance_with_caching
-    
+
 cwl_v1.1_kubernetes:
   stage: main_tests
   only: []
@@ -162,7 +162,7 @@ cwl_v1.1_kubernetes:
     - mkdir -p ${TOIL_WORKDIR}
     - make test tests=src/toil/test/cwl/cwlTest.py::CWLv11Test::test_kubernetes_cwl_conformance
     - make test tests=src/toil/test/cwl/cwlTest.py::CWLv11Test::test_kubernetes_cwl_conformance_with_caching
-    
+
 cwl_v1.2_kubernetes:
   stage: main_tests
   script:
@@ -187,12 +187,11 @@ wdl:
     - make test tests=src/toil/test/wdl/toilwdlTest.py  # needs java (default-jre) to run "GATK.jar"
     - make test tests=src/toil/test/wdl/builtinTest.py
 
-jobstore_and_provisioning:
+provisioning:
   stage: main_tests
   script:
     - pwd
     - virtualenv -p ${MAIN_PYTHON_PKG} venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
-    - make test tests=src/toil/test/jobStores/jobStoreTest.py
     - make test tests=src/toil/test/sort/sortTest.py
     - make test tests=src/toil/test/provisioners/aws/awsProvisionerTest.py
     - make test tests=src/toil/test/provisioners/clusterScalerTest.py
@@ -207,7 +206,7 @@ main:
     - virtualenv -p ${MAIN_PYTHON_PKG} venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages=htcondor
     - make test tests=src/toil/test/src
     - make test tests=src/toil/test/utils
-#    - make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testDocker
+    - make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testDocker
 
 appliance_build:
   stage: basic_tests

Makefile

@@ -120,7 +120,7 @@ clean_sdist:
 # Setting SET_OWNER_TAG will tag cloud resources so that UCSC's cloud murder bot won't kill them.
 test: check_venv check_build_reqs
 	TRAVIS=true TOIL_OWNER_TAG="shared" \
-	    python -m pytest --durations=0 --log-level DEBUG --log-cli-level INFO -r s $(cov) $(tests)
+	    python -m pytest --durations=0 --log-level DEBUG --log-cli-level INFO -r s $(tests)
 
 
 # This target will skip building docker and all docker based tests

contrib/admin/cleanup_aws_resources.py

@@ -17,12 +17,25 @@
 import re
 import sys
 
+from typing import Optional
+
 pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))  # noqa
 sys.path.insert(0, pkg_root)  # noqa
 
-from src.toil.lib import aws
-from src.toil.lib.aws.utils import delete_iam_role, delete_iam_instance_profile, delete_s3_bucket, delete_sdb_domain
+from src.toil.lib.aws.iam import delete_iam_role, delete_iam_instance_profile
+from src.toil.lib.aws.s3 import delete_bucket
 from src.toil.lib.generatedEC2Lists import regionDict
+from src.toil.lib.retry import retry
+from src.toil.lib.aws.credentials import client, resource
+
+try:
+    from boto.exception import BotoServerError
+    from mypy_boto3_s3 import S3ServiceResource
+    from mypy_boto3_s3.literals import BucketLocationConstraintType
+    from mypy_boto3_s3.service_resource import Bucket
+except ImportError:
+    BotoServerError = None  # type: ignore
+    # AWS/boto extra is not installed
 
 # put us-west-2 first as our default test region; that way anything with a universal region shows there
 regions = ['us-west-2'] + [region for region in regionDict if region != 'us-west-2']
@@ -39,6 +52,14 @@
                                           'toil-preserve-file-permissions-tests']  # test infra; never delete
 
 
+# this is only here as a clean up tool; we no longer use sdb and this will eventually be removed
+@retry(errors=[BotoServerError])
+def delete_sdb_domain(sdb_domain_name: str, region: Optional[str] = None) -> None:
+    sdb_client = client("sdb", region_name=region)
+    sdb_client.delete_domain(DomainName=sdb_domain_name)
+    print(f'SBD Domain: "{sdb_domain_name}" successfully deleted.')
+
+
 def contains_uuid(string):
     """
     Determines if a string contains a pattern like: '28064c76-a491-43e7-9b50-da424f920354',
@@ -68,11 +89,7 @@ def contains_toil_test_patterns(string):
 
 
 def matches(resource_name):
-    if resource_name.endswith('--files') or resource_name.endswith('--jobs') or resource_name.endswith('_toil'):
-        if contains_toil_test_patterns(resource_name):
-            return resource_name
-
-    if resource_name.startswith('import-export-test-'):
+    if resource_name.endswith('--toil'):
         return resource_name
 
 
@@ -81,7 +98,7 @@ def find_buckets_to_cleanup(include_all, match):
     for region in regions:
         print(f'\n[{region}] Buckets:')
         try:
-            s3_resource = aws.resource('s3', region_name=region)
+            s3_resource = resource('s3', region_name=region)
             buckets_in_region = find_buckets_in_region(s3_resource, include_all, match)
             new_buckets = [b for b in buckets_in_region if b not in buckets]
             print('    ' + '\n    '.join(new_buckets))
@@ -101,7 +118,7 @@ def find_sdb_domains_to_cleanup(include_all, match):
     for region in regions:
         print(f'\n[{region}] SimpleDB Domains:')
         try:
-            sdb_client = aws.client('sdb', region_name=region)
+            sdb_client = client('sdb', region_name=region)
             domains_in_region = find_sdb_domains_in_region(sdb_client, include_all, match)
             new_domains = [b for b in domains_in_region if b not in sdb_domains]
             print('    ' + '\n    '.join(new_domains))
@@ -122,7 +139,7 @@ def find_iam_roles_to_cleanup(include_all, match):
     for region in regions:
         print(f'\n[{region}] IAM Roles:')
         try:
-            iam_client = aws.client('iam', region_name=region)
+            iam_client = client('iam', region_name=region)
             roles_in_region = find_iam_roles_in_region(iam_client, include_all, match)
 
             new_roles = [b for b in roles_in_region if b not in iam_roles]
@@ -139,8 +156,8 @@ def find_instance_profile_names_to_cleanup(include_all, match):
     for region in regions:
         print(f'\n[{region}] IAM Instance Profiles:')
         try:
-            iam_resource = aws.resource('iam', region_name=region)
-            iam_client = aws.client('iam')
+            iam_resource = resource('iam', region_name=region)
+            iam_client = client('iam')
             instance_profiles_in_region = find_instance_profile_names_in_region(iam_client, include_all, match)
 
             new_instance_profiles = [b for b in instance_profiles_in_region if b not in instance_profiles]
@@ -252,7 +269,7 @@ def main(argv):
 
     options = parser.parse_args(argv)
 
-    account_name = aws.client('iam').list_account_aliases()['AccountAliases'][0]
+    account_name = client('iam').list_account_aliases()['AccountAliases'][0]
     print(f'\n\nNow running for AWS account: {account_name}.')
 
     match = [m.strip() for m in options.match.split(',') if m.strip()]
@@ -278,7 +295,8 @@ def main(argv):
                 if response.lower() in ('y', 'yes'):
                     print('\nOkay, now deleting...')
                     for bucket, region in buckets.items():
-                        delete_s3_bucket(bucket, region)
+                        s3_resource = resource('s3', region_name=region)
+                        delete_bucket(s3_resource, bucket)
                     print('S3 Bucket Deletions Successful.')
 
     if not options.skip_sdb:

src/toil/__init__.py

@@ -19,6 +19,7 @@
 import subprocess
 import sys
 import time
+import json
 from datetime import datetime
 
 import requests
@@ -32,73 +33,6 @@
 log = logging.getLogger(__name__)
 
 
-def which(cmd, mode=os.F_OK | os.X_OK, path=None):
-    """
-    Copy-pasted in from python3.6's shutil.which().
-
-    Given a command, mode, and a PATH string, return the path which
-    conforms to the given mode on the PATH, or None if there is no such
-    file.
-
-    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
-    of os.environ.get("PATH"), or can be overridden with a custom search
-    path.
-
-    """
-
-    # Check that a given file can be accessed with the correct mode.
-    # Additionally check that `file` is not a directory, as on Windows
-    # directories pass the os.access check.
-    def _access_check(fn, mode):
-        return (os.path.exists(fn) and os.access(fn, mode)
-                and not os.path.isdir(fn))
-
-    # If we're given a path with a directory part, look it up directly rather
-    # than referring to PATH directories. This includes checking relative to the
-    # current directory, e.g. ./script
-    if os.path.dirname(cmd):
-        if _access_check(cmd, mode):
-            return cmd
-        return None
-
-    if path is None:
-        path = os.environ.get("PATH", os.defpath)
-    if not path:
-        return None
-    path = path.split(os.pathsep)
-
-    if sys.platform == "win32":
-        # The current directory takes precedence on Windows.
-        if not os.curdir in path:
-            path.insert(0, os.curdir)
-
-        # PATHEXT is necessary to check on Windows.
-        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
-        # See if the given file matches any of the expected path extensions.
-        # This will allow us to short circuit when given "python.exe".
-        # If it does match, only test that one, otherwise we have to try
-        # others.
-        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
-            files = [cmd]
-        else:
-            files = [cmd + ext for ext in pathext]
-    else:
-        # On other platforms you don't have things like PATHEXT to tell you
-        # what file suffixes are executable, so just pass on cmd as-is.
-        files = [cmd]
-
-    seen = set()
-    for dir in path:
-        normdir = os.path.normcase(dir)
-        if not normdir in seen:
-            seen.add(normdir)
-            for thefile in files:
-                name = os.path.join(dir, thefile)
-                if _access_check(name, mode):
-                    return name
-    return None
-
-
 def toilPackageDirPath():
     """
     Returns the absolute path of the directory that corresponds to the top-level toil package.
@@ -192,8 +126,8 @@ def applianceSelf(forceDockerAppliance=False):
 
     if forceDockerAppliance:
         return appliance
-    else:
-        return checkDockerImageExists(appliance=appliance)
+
+    return checkDockerImageExists(appliance=appliance)
 
 
 def customDockerInitCmd():
@@ -249,14 +183,9 @@ def lookupEnvVar(name, envName, defaultValue):
     :return: the value of the environment variable or the default value the variable is not set
     :rtype: str
     """
-    try:
-        value = os.environ[envName]
-    except KeyError:
-        log.info('Using default %s of %s as %s is not set.', name, defaultValue, envName)
-        return defaultValue
-    else:
-        log.info('Overriding %s of %s with %s from %s.', name, defaultValue, value, envName)
-        return value
+    value = os.environ.get(envName, defaultValue)
+    log.info(f'Setting {name} to "{value}".  (Set with: "{envName}"; Default: "{defaultValue}")')
+    return value
 
 
 def checkDockerImageExists(appliance):
@@ -423,7 +352,7 @@ def logProcessContext(config):
     # toil.version (module) and Sphinx doesn't like that.
     from toil.version import version
     log.info("Running Toil version %s on host %s.", version, socket.gethostname())
-    log.debug("Configuration: %s", config.__dict__)
+    log.debug("Configuration: %s", json.dumps(config.__dict__, indent=4))
 
 
 try:
@@ -496,7 +425,7 @@ def __init__(self, name, access_key=None, secret_key=None,
                 # We will backend into a boto3 resolver for getting credentials.
                 # Make sure to enable boto3's own caching, so we can share that
                 # cache with pure boto3 code elsewhere in Toil.
-                # Keep synced with toil.lib.ec2.establish_boto3_session
+                # Keep synced with the session used in toil.lib.aws.credentials
                 self._boto3_resolver = create_credential_resolver(Session(profile=profile_name), cache=JSONFileCache())
             else:
                 # We will use the normal flow

src/toil/batchSystems/mesos/test/__init__.py

@@ -100,7 +100,7 @@ def findMesosBinary(self, names):
                 sought = 'any binary in %s' % str(names)
 
             raise RuntimeError("Cannot find %s. Make sure Mesos is installed "
-                                "and it's 'bin' directory is present on the PATH." % sought)
+                               "and it's 'bin' directory is present on the PATH." % sought)
 
     class MesosMasterThread(MesosThread):
         def mesosCommand(self):

src/toil/common.py

@@ -32,7 +32,8 @@
 from toil.batchSystems.options import (add_all_batchsystem_options,
                                        set_batchsystem_config_defaults,
                                        set_batchsystem_options)
-from toil.lib.aws import zone_to_region
+
+from toil.lib.aws.util import zone_to_region
 from toil.lib.conversions import bytes2human, human2bytes
 from toil.lib.retry import retry
 from toil.provisioners import add_provisioner_options, cluster_factory, parse_node_types
@@ -817,8 +818,7 @@ def start(self, rootJob):
         self._assertContextManagerUsed()
         self.writePIDFile()
         if self.config.restart:
-            raise ToilRestartException('A Toil workflow can only be started once. Use '
-                                       'Toil.restart() to resume it.')
+            raise ToilRestartException('A Toil workflow can only be started once. Use Toil.restart() to resume it.')
 
         self._batchSystem = self.createBatchSystem(self.config)
         self._setupAutoDeployment(rootJob.getUserScript())
@@ -1433,7 +1433,6 @@ def getDirSizeRecursively(dirPath: str) -> int:
     :param str dirPath: A valid path to a directory or file.
     :return: Total size, in bytes, of the file or directory at dirPath.
     """
-
     # du is often faster than using os.lstat(), sometimes significantly so.
 
     # The call: 'du -s /some/path' should give the number of 512-byte blocks

src/toil/deferred.py

@@ -30,7 +30,7 @@
 class DeferredFunction(namedtuple('DeferredFunction', 'function args kwargs name module')):
     """
     >>> from collections import defaultdict
-    >>> df = DeferredFunction.create(defaultdict, None, {'x':1}, y=2)
+    >>> df = DeferredFunction.create_job(defaultdict, None, {'x':1}, y=2)
     >>> df
     DeferredFunction(defaultdict, ...)
     >>> df.invoke() == defaultdict(None, x=1, y=2)

src/toil/fileStores/__init__.py

@@ -34,6 +34,8 @@ def __init__(self, fileStoreID: str, size: int, executable: bool = False):
         super(FileID, self).__init__()
         self.size = size
         self.executable = executable
+        self.md5sum = None
+        self.etag = None
 
     def pack(self) -> str:
         """