diff --git a/.gitignore b/.gitignore index 4f0aa02..bd3a7de 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,10 @@ target/ # Scons .sconsign.dblite + +# doit +.doit.db +.doit.db.db + +# vim +*.swp diff --git a/.travis.yml b/.travis.yml index ab74573..f5ff061 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,5 +7,6 @@ before_install: install: - pip install -U pip setuptools - PIP_FIND_LINKS=~/travis-wheels/wheelhouse pip install -r dev-requirements.txt -e . + - pip install scons doit script: - py.test -v -x diff --git a/nbflow/doit.py b/nbflow/doit.py new file mode 100644 index 0000000..3174d10 --- /dev/null +++ b/nbflow/doit.py @@ -0,0 +1,58 @@ +from __future__ import print_function + +import json +import subprocess as sp +import sys + +from .extractor import DependencyExtractor +from ._version import __version__ + + +def build_notebook_cmd(notebook): + cmd = [ + "jupyter", "nbconvert", + "--log-level=ERROR", + "--ExecutePreprocessor.timeout=120", + "--execute", + "--inplace", + "--to", "notebook", + #"--output", notebook, + notebook + ] + return ' '.join(cmd) + + +def touch_cmds(target): + for t in target: + yield 'touch {0}'.format(str(t)) + + +def get_cmds(notebook, target): + yield build_notebook_cmd(notebook) + for cmd in touch_cmds(target): + yield cmd + + +def setup(directories): + parsed = sp.check_output([sys.executable, "-m", "nbflow"] + directories, + universal_newlines=True) + DEPENDENCIES = json.loads(parsed) + for script in DEPENDENCIES: + deps = DEPENDENCIES[script] + sources = deps['sources'] + if len(deps['targets']) == 0: + #targets = ['.phony_{}'.format(script)] + targets = [] + else: + targets = deps['targets'] + + name = 'build-notebook:{0}'.format(script) + file_dep = [script] + sources + actions = list(get_cmds(script, targets)) + + yield {'name': name, + 'actions': actions, + 'targets': targets, + 'file_dep': file_dep, + 'clean': True} + diff --git a/nbflow/example/dodo.py b/nbflow/example/dodo.py new file mode 100644 index 0000000..723c319 --- /dev/null +++ b/nbflow/example/dodo.py @@ -0,0 +1,6 @@ +import os +from nbflow import doit + +def task_nbflow(): + for task in doit.setup(['analyses']): + yield task diff --git a/nbflow/extractor.py b/nbflow/extractor.py index 6341382..b928f17 100644 --- a/nbflow/extractor.py +++ b/nbflow/extractor.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import os import glob import json @@ -57,11 +59,10 @@ def get_dependencies(self, dirnames): sources = [self.resolve_path(filename, x) for x in params['__depends__']] targets = params['__dest__'] - if not hasattr(targets, '__iter__'): - if targets is None: - targets = [] - else: - targets = [targets] + if targets is None: + targets = [] + elif isinstance(targets, str): + targets = [targets] targets = [self.resolve_path(filename, x) for x in targets] dependencies[os.path.join(dirname, '{}.ipynb'.format(modname))] = { diff --git a/nbflow/tests/test_nbflow.py b/nbflow/tests/test_nbflow.py index 4a48f3f..a09087c 100644 --- a/nbflow/tests/test_nbflow.py +++ b/nbflow/tests/test_nbflow.py @@ -12,7 +12,7 @@ def test_nbflow_no_args(temp_cwd): def test_example(temp_cwd): # copy example files - root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "example")) + root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "scons-example")) shutil.copytree(os.path.join(root, "analyses"), "analyses") shutil.copy(os.path.join(root, "SConstruct"), "SConstruct") clear_notebooks("analyses") diff --git a/requirements.txt b/requirements.txt index ee81e4d..e711ee4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ traitlets nbconvert==4.1 -scons -ipykernel \ No newline at end of file +ipykernel