[enhancement] Cache CircleCI tests, add isort, black tests (#60)

Author: vedanuj

.circleci/config.yml

@@ -1,38 +1,105 @@
+# Python CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-python/ for more details
+#
+# Adopted from
+# https://github.com/facebookresearch/detectron2/blob/master/.circleci/config.yml
+
 version: 2
+
+# -------------------------------------------------------------------------------------
+# Environments to run the jobs in
+# -------------------------------------------------------------------------------------
+cpu: &cpu
+  docker:
+    - image: circleci/python:3.7
+  resource_class: medium
+
+# -------------------------------------------------------------------------------------
+# Re-usable commands
+# -------------------------------------------------------------------------------------
+setup_venv: &setup_venv
+  - run:
+      name: Setup Virtual Env
+      working_directory: ~/
+      command: |
+        python -m venv ~/venv
+        echo ". ~/venv/bin/activate" >> $BASH_ENV
+        . ~/venv/bin/activate
+        python --version
+        which python
+        which pip
+        pip install --upgrade pip
+
+install_dep: &install_dep
+  - run:
+      name: Install Dependencies
+      command: |
+        pip install --progress-bar off -r requirements.txt
+        pip install --progress-bar off flake8
+        pip install --progress-bar off black
+        pip install --progress-bar off isort
+
+install_repo: &install_repo
+  - run:
+      name: Install Repository
+      command: |
+        python setup.py build develop
+
+run_unittests: &run_unittests
+  - run:
+      name: Run Unit Tests
+      command: |
+        cd tests
+        pytest -v .
+
+run_flake8: &run_flake8
+  - run:
+      name: Run Linter (flake8)
+      command: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings
+        flake8 . --count --exit-zero --max-complexity=18 --max-line-length=88 --statistics
+
+# -------------------------------------------------------------------------------------
+# Jobs to run
+# -------------------------------------------------------------------------------------
+
 jobs:
   build:
-    docker:
-      - image: circleci/python:3.7
+    <<: *cpu
+
     working_directory: ~/pythia
+
     steps:
       - checkout
+      - <<: *setup_venv
+
+      # Cache the venv directory that contains dependencies
+      - restore_cache:
+          keys:
+            - cache-key-{{ checksum "requirements.txt"}}-{{ checksum "setup.py"}}
+
+      - <<: *install_dep
+
+      - save_cache:
+          paths:
+            - ~/venv
+          key: cache-key-{{ checksum "requirements.txt"}}-{{ checksum "setup.py"}}
+
+      - <<: *install_repo
+
       - run:
-          name: Install Dependencies
+          name: Run Linter (isort)
           command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install --upgrade pip
-            python setup.py develop
+            isort -c -sp .
+
       - run:
-          name: Run linter (flake8)
+          name: Run Linter (black)
           command: |
-            . venv/bin/activate
-            pip install flake8
-            # stop the build if there are Python syntax errors or undefined names
-            flake8 . --count --exclude=./venv --select=E9,F63,F72,F82 --show-source --statistics
-            # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
-            flake8 . --count --exclude=./venv --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - run:
-          name: Download required data
-          command: |
-            mkdir data
-            cd data
-            wget http://dl.fbaipublicfiles.com/pythia/data/vocab.tar.gz
-            tar xf vocab.tar.gz
-            cd ..
-      - run:
-          name: Run Tests
-          command: |
-            . venv/bin/activate
-            cd tests
-            pytest -v .
+            black --check -l 88 .
+
+      - <<: *run_flake8
+
+      - <<: *run_unittests

.editorconfig

@@ -9,4 +9,4 @@ indent_style = space
 indent_size = 4
 
 [*.md]
-trim_trailing_whitespace = false
+trim_trailing_whitespace = false

docs/source/conf.py

@@ -32,46 +32,47 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 
-import sphinx_rtd_theme
 import recommonmark
+import sphinx_rtd_theme
 from recommonmark.transform import AutoStructify
 
-
-extensions = ['sphinx.ext.autodoc',
-    'sphinx.ext.doctest',
-    'sphinx.ext.todo',
-    'sphinx.ext.coverage',
-    'sphinx.ext.mathjax',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.napoleon',
-    'sphinx.ext.intersphinx',
-    'recommonmark']
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.doctest",
+    "sphinx.ext.todo",
+    "sphinx.ext.coverage",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.intersphinx",
+    "recommonmark",
+]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
-source_suffix = ['.rst', '.md']
+source_suffix = [".rst", ".md"]
 # source_suffix = '.rst'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = 'Pythia'
-copyright = '2019, Facebook AI Research'
-author = 'Facebook AI Research'
+project = "Pythia"
+copyright = "2019, Facebook AI Research"
+author = "Facebook AI Research"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '0.3'
+version = "0.3"
 # The full version, including alpha/beta/rc tags.
-release = '0.3'
+release = "0.3"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -83,10 +84,10 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
@@ -97,7 +98,7 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
@@ -109,25 +110,25 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
 #
 # This is required for the alabaster theme
 # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
 html_sidebars = {
-    '**': [
-        'relations.html',  # needs 'show_related': True theme option to display
-        'searchbox.html',
+    "**": [
+        "relations.html",  # needs 'show_related': True theme option to display
+        "searchbox.html",
     ]
 }
 
 
 # -- Options for HTMLHelp output ------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'pythiadoc'
+htmlhelp_basename = "pythiadoc"
 
 
 # -- Options for LaTeX output ---------------------------------------------
@@ -136,15 +137,12 @@
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -154,19 +152,21 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'pythia.tex', 'Pythia Documentation',
-     'Facebook AI Research', 'manual'),
+    (
+        master_doc,
+        "pythia.tex",
+        "Pythia Documentation",
+        "Facebook AI Research",
+        "manual",
+    ),
 ]
 
 
 # -- Options for manual page output ---------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'pythia', 'Pythia Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, "pythia", "Pythia Documentation", [author], 1)]
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -175,17 +175,27 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'pythia', 'pythia Documentation',
-     author, 'pythia', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "pythia",
+        "pythia Documentation",
+        author,
+        "pythia",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 github_doc_root = "https://github.com/facebookresearch/pythia/blob/v0.3/"
 
 # At the bottom of conf.py
 def setup(app):
-    app.add_config_value('recommonmark_config', {
-            'url_resolver': lambda url: github_doc_root + url,
-            'auto_toc_tree_section': 'Contents',
-            }, True)
+    app.add_config_value(
+        "recommonmark_config",
+        {
+            "url_resolver": lambda url: github_doc_root + url,
+            "auto_toc_tree_section": "Contents",
+        },
+        True,
+    )
     app.add_transform(AutoStructify)

docs/source/utils/text_utils.rst

@@ -2,4 +2,4 @@ utils.text_utils
 ===============
 
 .. automodule:: pythia.utils.text_utils
-  :members:
+  :members:

pythia/datasets/__init__.py

@@ -1,6 +1,6 @@
 # Copyright (c) Facebook, Inc. and its affiliates.
+from .base_dataset import BaseDataset
 from .base_dataset_builder import BaseDatasetBuilder
 from .multi_dataset import MultiDataset
-from .base_dataset import BaseDataset
 
 __all__ = ["BaseDataset", "BaseDatasetBuilder", "MultiDataset"]

pythia/datasets/builders/clevr/builder.py

@@ -4,8 +4,8 @@
 import zipfile
 from collections import Counter
 
-from pythia.common.registry import registry
 from pythia.common.constants import CLEVR_DOWNLOAD_URL
+from pythia.common.registry import registry
 from pythia.datasets.base_dataset_builder import BaseDatasetBuilder
 from pythia.datasets.builders.clevr.dataset import CLEVRDataset
 from pythia.utils.general import download_file, get_pythia_root

pythia/datasets/builders/coco/masked_builder.py

@@ -1,7 +1,6 @@
 from pythia.common.registry import registry
 from pythia.datasets.builders.coco.builder import COCOBuilder
 
-
 from .masked_dataset import MaskedCOCODataset
 
 

pythia/datasets/builders/conceptual_captions/masked_builder.py

@@ -2,7 +2,6 @@
 from pythia.common.registry import registry
 from pythia.datasets.builders.coco import MaskedCOCOBuilder
 
-
 from .masked_dataset import MaskedConceptualCaptionsDataset
 
 

pythia/datasets/builders/mmimdb/builder.py

@@ -6,8 +6,8 @@
 #
 
 from pythia.common.registry import registry
-from pythia.datasets.builders.vqa2.builder import VQA2Builder
 from pythia.datasets.builders.mmimdb.dataset import MMIMDbDataset
+from pythia.datasets.builders.vqa2.builder import VQA2Builder
 
 
 @registry.register_builder("mmimdb")