Add Test Cases for Python 3.7

.travis.yml

@@ -8,6 +8,9 @@ matrix:
     env: TOX_ENV=py27
   - python: 3.5
     env: TOX_ENV=py35
+  - python: 3.7
+    env: TOX_ENV=py37
+    dist: xenial
   - python: 2.7
     env: TOX_ENV=flake8
   - python: 2.7
@@ -18,7 +21,7 @@ before_install:
   - tsc --module amd --noImplicitAny --outdir datalab/notebook/static datalab/notebook/static/*.ts
   # We use tox for actually running tests.
   - pip install --upgrade pip tox
-  
+
 script:
   # tox reads its configuration from tox.ini.
   - tox -e $TOX_ENV

datalab/utils/__init__.py

@@ -12,7 +12,7 @@
 
 """Google Cloud Platform library - Internal Helpers."""
 
-from ._async import async, async_function, async_method
+from ._async import async_, async_function, async_method
 from ._gcp_job import GCPJob
 from ._http import Http, RequestException
 from ._iterator import Iterator
@@ -24,7 +24,7 @@
 from ._utils import print_exception_with_last_stack, get_item, compare_datetimes, \
     pick_unused_port, is_http_running_on, gcs_copy_file
 
-__all__ = ['async', 'async_function', 'async_method', 'GCPJob', 'Http', 'RequestException',
+__all__ = ['async_', 'async_function', 'async_method', 'GCPJob', 'Http', 'RequestException',
            'Iterator', 'Job', 'JobError', 'JSONEncoder', 'LRUCache', 'LambdaJob', 'DataflowJob',
            'print_exception_with_last_stack', 'get_item', 'compare_datetimes', 'pick_unused_port',
            'is_http_running_on', 'gcs_copy_file']

datalab/utils/_async.py

@@ -23,7 +23,7 @@
 from future.utils import with_metaclass
 
 
-class async(with_metaclass(abc.ABCMeta, object)):
+class async_(with_metaclass(abc.ABCMeta, object)):
   """ Base class for async_function/async_method. Creates a wrapped function/method that will
       run the original function/method on a thread pool worker thread and return a Job instance
       for monitoring the status of the thread.
@@ -55,27 +55,27 @@ def __call__(self, *args, **kwargs):
     return _job.Job(future=self.executor.submit(self._call, *args, **kwargs))
 
 
-class async_function(async):
+class async_function(async_):
   """ This decorator can be applied to any static function that makes blocking calls to create
       a modified version that creates a Job and returns immediately; the original
       method will be called on a thread pool worker thread.
   """
 
   def _call(self, *args, **kwargs):
     # Call the wrapped method.
-    return self._function(*async._preprocess_args(*args), **async._preprocess_kwargs(**kwargs))
+    return self._function(*async_._preprocess_args(*args), **async_._preprocess_kwargs(**kwargs))
 
 
-class async_method(async):
+class async_method(async_):
   """ This decorator can be applied to any class instance method that makes blocking calls to create
       a modified version that creates a Job and returns immediately; the original method will be
       called on a thread pool worker thread.
   """
 
   def _call(self, *args, **kwargs):
     # Call the wrapped method.
-    return self._function(self.obj, *async._preprocess_args(*args),
-                          **async._preprocess_kwargs(**kwargs))
+    return self._function(self.obj, *async_._preprocess_args(*args),
+                          **async_._preprocess_kwargs(**kwargs))
 
   def __get__(self, instance, owner):
     # This is important for attribute inheritance and setting self.obj so it can be

datalab/utils/_lambda_job.py

@@ -30,7 +30,7 @@ def __init__(self, fn, job_id, *args, **kwargs):
       job_id: an optional ID for the job. If None, a UUID will be generated.
     """
     super(LambdaJob, self).__init__(job_id)
-    self._future = _async.async.executor.submit(fn, *args, **kwargs)
+    self._future = _async.async_.executor.submit(fn, *args, **kwargs)
 
   def __repr__(self):
     """Returns a representation for the job for showing in the notebook.

datalab/utils/commands/_utils.py

@@ -30,6 +30,7 @@
   import pandas_profiling
 except ImportError:
   pass
+import six
 import sys
 import types
 import yaml
@@ -323,6 +324,8 @@ def parse_config(config, env, as_dict=True):
     config = {}
   elif stripped[0] == '{':
     config = json.loads(config)
+  elif six.PY3:
+    config = yaml.load(config, Loader=yaml.FullLoader)
   else:
     config = yaml.load(config)
   if as_dict:

google/datalab/utils/__init__.py

@@ -12,7 +12,7 @@
 
 """Google Cloud Platform library - Internal Helpers."""
 
-from ._async import async, async_function, async_method
+from ._async import async_, async_function, async_method
 from ._http import Http, RequestException
 from ._iterator import Iterator
 from ._json_encoder import JSONEncoder
@@ -23,7 +23,7 @@
     pick_unused_port, is_http_running_on, gcs_copy_file, python_portable_string
 
 
-__all__ = ['async', 'async_function', 'async_method', 'Http', 'RequestException', 'Iterator',
+__all__ = ['async_', 'async_function', 'async_method', 'Http', 'RequestException', 'Iterator',
            'JSONEncoder', 'LRUCache', 'LambdaJob', 'DataflowJob',
            'print_exception_with_last_stack', 'get_item', 'compare_datetimes', 'pick_unused_port',
            'is_http_running_on', 'gcs_copy_file', 'python_portable_string']

google/datalab/utils/_async.py

@@ -23,7 +23,7 @@
 from future.utils import with_metaclass
 
 
-class async(with_metaclass(abc.ABCMeta, object)):
+class async_(with_metaclass(abc.ABCMeta, object)):
   """ Base class for async_function/async_method. Creates a wrapped function/method that will
       run the original function/method on a thread pool worker thread and return a Job instance
       for monitoring the status of the thread.
@@ -55,27 +55,27 @@ def __call__(self, *args, **kwargs):
     return Job(future=self.executor.submit(self._call, *args, **kwargs))
 
 
-class async_function(async):
+class async_function(async_):
   """ This decorator can be applied to any static function that makes blocking calls to create
       a modified version that creates a Job and returns immediately; the original
       method will be called on a thread pool worker thread.
   """
 
   def _call(self, *args, **kwargs):
     # Call the wrapped method.
-    return self._function(*async._preprocess_args(*args), **async._preprocess_kwargs(**kwargs))
+    return self._function(*async_._preprocess_args(*args), **async_._preprocess_kwargs(**kwargs))
 
 
-class async_method(async):
+class async_method(async_):
   """ This decorator can be applied to any class instance method that makes blocking calls to create
       a modified version that creates a Job and returns immediately; the original method will be
       called on a thread pool worker thread.
   """
 
   def _call(self, *args, **kwargs):
     # Call the wrapped method.
-    return self._function(self.obj, *async._preprocess_args(*args),
-                          **async._preprocess_kwargs(**kwargs))
+    return self._function(self.obj, *async_._preprocess_args(*args),
+                          **async_._preprocess_kwargs(**kwargs))
 
   def __get__(self, instance, owner):
     # This is important for attribute inheritance and setting self.obj so it can be

google/datalab/utils/_lambda_job.py

@@ -30,7 +30,7 @@ def __init__(self, fn, job_id, *args, **kwargs):
       job_id: an optional ID for the job. If None, a UUID will be generated.
     """
     super(LambdaJob, self).__init__(job_id)
-    self._future = _async.async.executor.submit(fn, *args, **kwargs)
+    self._future = _async.async_.executor.submit(fn, *args, **kwargs)
 
   def __repr__(self):
     """Returns a representation for the job for showing in the notebook.

google/datalab/utils/commands/_utils.py

@@ -30,6 +30,7 @@
   import pandas_profiling
 except ImportError:
   pass
+import six
 import sys
 import yaml
 
@@ -328,6 +329,8 @@ def parse_config(config, env, as_dict=True):
     config = {}
   elif stripped[0] == '{':
     config = json.loads(config)
+  elif six.PY3:
+    config = yaml.load(config, Loader=yaml.FullLoader)
   else:
     config = yaml.load(config)
   if as_dict:
@@ -373,6 +376,8 @@ def parse_config_for_selected_keys(content, keys):
     return {}, None
   elif stripped[0] == '{':
     config = json.loads(content)
+  elif six.PY3:
+    config = yaml.load(content, Loader=yaml.FullLoader)
   else:
     config = yaml.load(content)
 

setup.py

@@ -78,6 +78,7 @@
   classifiers=[
       "Programming Language :: Python",
       "Programming Language :: Python :: 2",
+      "Programming Language :: Python :: 3",
       "Development Status :: 4 - Beta",
       "Environment :: Other Environment",
       "Intended Audience :: Developers",

tests/_util/commands_tests.py

@@ -95,7 +95,13 @@ def test_subcommand_line_cell(self):
                               'string4: value4\nflag1: false')
     self.assertEqual(args, {'string1': 'value1', 'string2': 'value2', 'string3': 'value3',
                             'command': 'subcommand2', 'flag1': False})
-    self.assertEqual(yaml.load(cell), {'string3': 'value3', 'string4': 'value4'})
+    if six.PY3:
+      self.assertEqual(
+        yaml.load(cell, Loader=yaml.FullLoader),
+        {'string3': 'value3', 'string4': 'value4'}
+      )
+    else:
+      self.assertEqual(yaml.load(cell), {'string3': 'value3', 'string4': 'value4'})
 
     # Regular arg and cell arg cannot be the same name.
     with self.assertRaises(ValueError):
@@ -133,7 +139,10 @@ def test_subcommand_var_replacement(self):
     args, cell = parser.parse('subcommand1 --string1 $var1', 'a: b\nstring2: $var2', namespace)
     self.assertEqual(args,
                      {'string1': 'value1', 'string2': 'value2', 'flag1': False, 'dict1': None})
-    self.assertEqual(yaml.load(cell), {'a': 'b', 'string2': '$var2'})
+    if six.PY3:
+      self.assertEqual(yaml.load(cell, Loader=yaml.FullLoader), {'a': 'b', 'string2': '$var2'})
+    else:
+      self.assertEqual(yaml.load(cell), {'a': 'b', 'string2': '$var2'})
 
     cell = """
 dict1:

tests/pipeline/pipeline_tests.py

@@ -16,6 +16,7 @@
 import datetime
 import mock
 import re
+import six
 import unittest
 import yaml
 
@@ -262,7 +263,10 @@ def test_get_dag_definition(self):
                      'catchup=True, default_args=default_args)\n\n')
 
   def test_get_datetime_expr(self):
-    dag_dict = yaml.load(PipelineTest._test_pipeline_yaml_spec)
+    if six.PY3:
+      dag_dict = yaml.load(PipelineTest._test_pipeline_yaml_spec, Loader=yaml.FullLoader)
+    else:
+      dag_dict = yaml.load(PipelineTest._test_pipeline_yaml_spec)
     start = dag_dict.get('schedule').get('start')
     datetime_expr = pipeline.PipelineGenerator._get_datetime_expr_str(start)
 
@@ -277,7 +281,10 @@ def test_get_default_args(self):
     self.assertIn("'email': []", actual)
     self.assertIn("'owner': 'Google Cloud Datalab'", actual)
 
-    dag_dict = yaml.load(PipelineTest._test_pipeline_yaml_spec)
+    if six.PY3:
+      dag_dict = yaml.load(PipelineTest._test_pipeline_yaml_spec, Loader=yaml.FullLoader)
+    else:
+      dag_dict = yaml.load(PipelineTest._test_pipeline_yaml_spec)
     dag_dict['schedule']['retries'] = 5
     dag_dict['schedule']['email_on_retry'] = False
     dag_dict['schedule']['email_on_failure'] = False
@@ -302,7 +309,10 @@ def test_get_default_args(self):
     self.assertIn("'max_retry_delay': timedelta(seconds=15)", actual)
 
   def test_get_airflow_spec_with_default_schedule(self):
-    dag_dict = yaml.load(PipelineTest._test_pipeline_yaml_spec)
+    if six.PY3:
+      dag_dict = yaml.load(PipelineTest._test_pipeline_yaml_spec, Loader=yaml.FullLoader)
+    else:
+      dag_dict = yaml.load(PipelineTest._test_pipeline_yaml_spec)
     # We delete the schedule spec to test with defaults
     del dag_dict['schedule']
 

tox.ini

@@ -1,7 +1,7 @@
 [tox]
-# By default, we want to run tests for Python 2.7, Python 3.5, and run our
-# flake8 checks.
-envlist = py27,py35,flake8,coveralls
+# By default, we want to run tests for Python 2.7, Python 3.5, Python 3.7,
+# and run our flake8 checks.
+envlist = py27,py35,py37,flake8,coveralls
 # If an interpreter is missing locally, skip it.
 skip_missing_interpreters = true
 
@@ -10,9 +10,9 @@ skip_missing_interpreters = true
 # need them to run our tests suite.
 #
 # tox always installs the current package, so there's no need to list it here.
-deps = apache-airflow==1.9.0
+deps = apache-airflow==1.10.9
        dill==0.2.6
-       tensorflow==1.8.0
+       tensorflow==1.14.0
        lime==0.1.1.23
        xgboost==0.6a2
        # Dropping this seems to cause problems with conda in some cases.