From 375a2f5147f69d5616e6bd1f4f9bee7304cd1773 Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Tue, 30 May 2017 21:36:41 -0700 Subject: [PATCH 01/14] update .gitignore --- .gitignore | 1 + .../multiverso_python.egg-info/PKG-INFO | 42 +++++++++++++++++++ .../multiverso_python.egg-info/SOURCES.txt | 16 +++++++ .../dependency_links.txt | 1 + .../multiverso_python.egg-info/not-zip-safe | 1 + .../multiverso_python.egg-info/requires.txt | 2 + .../multiverso_python.egg-info/top_level.txt | 1 + include/multiverso/updater/dcasgd | 1 - 8 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 binding/python/multiverso_python.egg-info/PKG-INFO create mode 100644 binding/python/multiverso_python.egg-info/SOURCES.txt create mode 100644 binding/python/multiverso_python.egg-info/dependency_links.txt create mode 100644 binding/python/multiverso_python.egg-info/not-zip-safe create mode 100644 binding/python/multiverso_python.egg-info/requires.txt create mode 100644 binding/python/multiverso_python.egg-info/top_level.txt delete mode 160000 include/multiverso/updater/dcasgd diff --git a/.gitignore b/.gitignore index 5c1678f..156ffd6 100644 --- a/.gitignore +++ b/.gitignore @@ -270,3 +270,4 @@ _Pvt_Extensions # Python *.pyc +*.egg diff --git a/binding/python/multiverso_python.egg-info/PKG-INFO b/binding/python/multiverso_python.egg-info/PKG-INFO new file mode 100644 index 0000000..45a36bb --- /dev/null +++ b/binding/python/multiverso_python.egg-info/PKG-INFO @@ -0,0 +1,42 @@ +Metadata-Version: 1.1 +Name: multiverso-python +Version: 0.0.1 +Summary: Multiverso is a parameter server framework for distributed machine learning. This package can leverage multiple machines and GPUs to speed up the python programs. +Home-page: https://github.com/Microsoft/multiverso +Author: Microsoft +Author-email: UNKNOWN +License: MIT +Description: # Multiverso Python/Theano/Lasagne Binding + + + ## Introduction + Multiverso is a parameter server framework for distributed machine learning. This package can leverage multiple machines and GPUs to speed up the python programs. + + + ## Installation + + 1. (For GPU support only) Install CUDA, cuDNN according to this [guide](https://github.com/Microsoft/fb.resnet.torch/blob/multiverso/INSTALL.md). You just need finish the steps before [Install Torch](https://github.com/Microsoft/fb.resnet.torch/blob/multiverso/INSTALL.md#install-torch). + 1. Install the multiverso + * On linux: Please follow the [README](https://github.com/Microsoft/multiverso/blob/master/README.md#build) to build and install multiverso. + * On windows: You need MSBuild.exe installed and make sure your system can find it in the $PATH. Then you should run [build_dll.bat](https://github.com/Microsoft/multiverso/blob/master/src/build_dll.bat) to build the .dll file and install the .dll. There isn't auto-installer for windows now, so you have to copy the .dll to either system $PATH or the multiverso package folder. + 1. Install the requirements + * `gfortran` is required by scipy. e.g. you can install it by `sudo apt-get install gfortran` on ubuntu. + * (Optional) You need python-nose to run the unit tests. e.g. you can install it by `sudo apt-get install python-nose` on ubuntu. + 1. Install python binding with the command `sudo python setup.py install` + + + ## Run Unit Tests + ``` + nosetests + ``` + + + ## Documentation + * [Tutorial](https://github.com/Microsoft/multiverso/wiki/How-to-write-python-code-with-multiverso) + * Api documents are written as docstrings in the python source code. + * [Benchmark](https://github.com/Microsoft/multiverso/wiki/Multiverso-Python-Binding-Benchmark) + +Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Science/Research +Classifier: Programming Language :: Python :: 2 diff --git a/binding/python/multiverso_python.egg-info/SOURCES.txt b/binding/python/multiverso_python.egg-info/SOURCES.txt new file mode 100644 index 0000000..e531455 --- /dev/null +++ b/binding/python/multiverso_python.egg-info/SOURCES.txt @@ -0,0 +1,16 @@ +setup.py +multiverso/__init__.py +multiverso/api.py +multiverso/tables.py +multiverso/utils.py +multiverso/theano_ext/__init__.py +multiverso/theano_ext/param_manager.py +multiverso/theano_ext/sharedvar.py +multiverso/theano_ext/lasagne_ext/__init__.py +multiverso/theano_ext/lasagne_ext/param_manager.py +multiverso_python.egg-info/PKG-INFO +multiverso_python.egg-info/SOURCES.txt +multiverso_python.egg-info/dependency_links.txt +multiverso_python.egg-info/not-zip-safe +multiverso_python.egg-info/requires.txt +multiverso_python.egg-info/top_level.txt \ No newline at end of file diff --git a/binding/python/multiverso_python.egg-info/dependency_links.txt b/binding/python/multiverso_python.egg-info/dependency_links.txt new file mode 100644 index 0000000..250ad83 --- /dev/null +++ b/binding/python/multiverso_python.egg-info/dependency_links.txt @@ -0,0 +1 @@ +https://github.com/Lasagne/Lasagne/tarball/master#egg=lasagne-0.2.dev1 diff --git a/binding/python/multiverso_python.egg-info/not-zip-safe b/binding/python/multiverso_python.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/binding/python/multiverso_python.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/binding/python/multiverso_python.egg-info/requires.txt b/binding/python/multiverso_python.egg-info/requires.txt new file mode 100644 index 0000000..28024b0 --- /dev/null +++ b/binding/python/multiverso_python.egg-info/requires.txt @@ -0,0 +1,2 @@ +theano>=0.8.2 +lasagne>=0.2.dev1 diff --git a/binding/python/multiverso_python.egg-info/top_level.txt b/binding/python/multiverso_python.egg-info/top_level.txt new file mode 100644 index 0000000..fe0aaae --- /dev/null +++ b/binding/python/multiverso_python.egg-info/top_level.txt @@ -0,0 +1 @@ +multiverso diff --git a/include/multiverso/updater/dcasgd b/include/multiverso/updater/dcasgd deleted file mode 160000 index 4bdd2c3..0000000 --- a/include/multiverso/updater/dcasgd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4bdd2c38702be6f443f9540f9d76d5190cf13e06 From bf7ae22cd8f32409a5faf83c3f51d0fc39ba22cd Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Wed, 31 May 2017 09:07:42 -0700 Subject: [PATCH 02/14] dist mnist --- binding/python/multiverso/api.py | 3 +- .../python/multiverso/torch_ext/__init__.py | 0 .../python/multiverso/torch_ext/torchmodel.py | 47 +++++++++++++++++++ .../multiverso_python.egg-info/SOURCES.txt | 2 + binding/python/setup.py | 2 +- 5 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 binding/python/multiverso/torch_ext/__init__.py create mode 100644 binding/python/multiverso/torch_ext/torchmodel.py diff --git a/binding/python/multiverso/api.py b/binding/python/multiverso/api.py index 849e51b..1a96177 100644 --- a/binding/python/multiverso/api.py +++ b/binding/python/multiverso/api.py @@ -9,7 +9,7 @@ mv_lib = Loader.get_lib() -def init(sync=False): +def init(sync=False, updater='sgd'): '''Initialize mutliverso. This should be called only once before training at the beginning of the @@ -29,6 +29,7 @@ def init(sync=False): args = [b""] # the first argument will be ignored. So we put a placeholder here if sync: args.append(b"-sync=true") + args.append(b"-updater=sgd") n = len(args) args_type = ctypes.c_char_p * n mv_lib.MV_Init(ctypes.pointer(ctypes.c_int(n)), args_type(*[ctypes.c_char_p(arg) for arg in args])) diff --git a/binding/python/multiverso/torch_ext/__init__.py b/binding/python/multiverso/torch_ext/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/binding/python/multiverso/torch_ext/torchmodel.py b/binding/python/multiverso/torch_ext/torchmodel.py new file mode 100644 index 0000000..82bd2e6 --- /dev/null +++ b/binding/python/multiverso/torch_ext/torchmodel.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# coding:utf8 + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms +from torch.autograd import Variable + +import numpy as np +import multiverso as mv + + +class MVTorchModel(object): + def __init__(self, tmobj): + assert(isinstance(tmobj, nn.Module)) + self._tmobj = tmobj + self._mv_params=[] + for param in self._tmobj.parameters(): + self._mv_params.append(mv.ArrayTableHandler(param.data.numpy().size, param.data.numpy().reshape((-1,)))) + mv.barrier() + self._last_mv_params=[] + for mv_param in self._mv_params: + self._last_mv_params.append(mv_param.get()) + for param, last_mv_param in zip(self._tmobj.parameters(), self._last_mv_params): + param=Variable(torch.from_numpy(last_mv_param.reshape(param.data.numpy().shape))) + + def mv_sync(self): + for mv_param, last_mv_param, param in zip(self._mv_params, self._last_mv_params, self._tmobj.parameters()): + mv_param.add(last_mv_param - param.data.numpy().reshape((-1,))) + + mv.barrier() + for mv_param, last_mv_param, param in zip(self._mv_params, self._last_mv_params, self._tmobj.parameters()): + last_mv_param = mv_param.get() + param=Variable(torch.from_numpy(last_mv_param.reshape(param.data.numpy().shape))) + + def __call__(self, *args, **kwargs): + return self._tmobj(*args, **kwargs) + + def __getattribute__(self, attr): + if attr in ['_tmobj', '_mv_params', '_last_mv_params']: + return object.__getattribute__(self, attr) + elif attr in ['mv_sync', '__call__']: + return getattr(MVTorchModel, attr).__get__(self) + else: + return getattr(self._tmobj, attr) diff --git a/binding/python/multiverso_python.egg-info/SOURCES.txt b/binding/python/multiverso_python.egg-info/SOURCES.txt index e531455..10ee279 100644 --- a/binding/python/multiverso_python.egg-info/SOURCES.txt +++ b/binding/python/multiverso_python.egg-info/SOURCES.txt @@ -8,6 +8,8 @@ multiverso/theano_ext/param_manager.py multiverso/theano_ext/sharedvar.py multiverso/theano_ext/lasagne_ext/__init__.py multiverso/theano_ext/lasagne_ext/param_manager.py +multiverso/torch_ext/__init__.py +multiverso/torch_ext/torchmodel.py multiverso_python.egg-info/PKG-INFO multiverso_python.egg-info/SOURCES.txt multiverso_python.egg-info/dependency_links.txt diff --git a/binding/python/setup.py b/binding/python/setup.py index f7b5604..a7108c3 100644 --- a/binding/python/setup.py +++ b/binding/python/setup.py @@ -16,7 +16,7 @@ def readme(): url='https://github.com/Microsoft/multiverso', author='Microsoft', license='MIT', - packages=['multiverso', 'multiverso.theano_ext', 'multiverso.theano_ext.lasagne_ext'], + packages=['multiverso', 'multiverso.torch_ext', 'multiverso.theano_ext', 'multiverso.theano_ext.lasagne_ext'], # TODO: The lasagne on pypi is too old. multiverso need some functions in # lasagne-0.2 which is not released yet. Please replace the dev version # with the stable release later. From a764c428d14a8e679236ab237d448aba6555f313 Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Wed, 31 May 2017 23:26:15 -0700 Subject: [PATCH 03/14] add dcasgd submodule --- .gitmodules | 2 +- include/multiverso/updater/dcasgd | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 160000 include/multiverso/updater/dcasgd diff --git a/.gitmodules b/.gitmodules index 8cb7c11..a8e2e14 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "include/multiverso/updater/dcasgd"] path = include/multiverso/updater/dcasgd - url = https://github.com/Microsoft/Delayed-Compensation-Asynchronous-Stochastic-Gradient-Descent-for-Multiverso.git + url = https://github.com/zhengsx/Delayed-Compensation-Asynchronous-Stochastic-Gradient-Descent-for-Multiverso.git diff --git a/include/multiverso/updater/dcasgd b/include/multiverso/updater/dcasgd new file mode 160000 index 0000000..4bdd2c3 --- /dev/null +++ b/include/multiverso/updater/dcasgd @@ -0,0 +1 @@ +Subproject commit 4bdd2c38702be6f443f9540f9d76d5190cf13e06 From 0040e7bec1988698d2ef2e4fc4b4aa6b91567dda Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Wed, 31 May 2017 23:26:52 -0700 Subject: [PATCH 04/14] cmake --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d63ca3..cdcf9bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,3 +43,4 @@ configure_file( add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) +add_definitions(-DENABLE_DCASGD) From a8bf835ca97180f654f04a7da17b57f5605ff2a7 Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Thu, 1 Jun 2017 01:23:56 -0700 Subject: [PATCH 05/14] nothing --- binding/python/multiverso/torch_ext/torchmodel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/binding/python/multiverso/torch_ext/torchmodel.py b/binding/python/multiverso/torch_ext/torchmodel.py index 82bd2e6..6fb82b5 100644 --- a/binding/python/multiverso/torch_ext/torchmodel.py +++ b/binding/python/multiverso/torch_ext/torchmodel.py @@ -31,6 +31,7 @@ def mv_sync(self): mv_param.add(last_mv_param - param.data.numpy().reshape((-1,))) mv.barrier() + for mv_param, last_mv_param, param in zip(self._mv_params, self._last_mv_params, self._tmobj.parameters()): last_mv_param = mv_param.get() param=Variable(torch.from_numpy(last_mv_param.reshape(param.data.numpy().shape))) From 935912dbba076a0757161caac0108339e3e2552c Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Thu, 1 Jun 2017 03:57:52 -0700 Subject: [PATCH 06/14] test example --- binding/python/examples/torch/mnist.py | 129 +++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 binding/python/examples/torch/mnist.py diff --git a/binding/python/examples/torch/mnist.py b/binding/python/examples/torch/mnist.py new file mode 100644 index 0000000..d77a9ad --- /dev/null +++ b/binding/python/examples/torch/mnist.py @@ -0,0 +1,129 @@ +from __future__ import print_function +import argparse +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms +from torch.autograd import Variable + +import numpy as np +import multiverso as mv +from multiverso.torch_ext import torchmodel + +mv.init(sync=True, updater='sgd') + +# Training settings +parser = argparse.ArgumentParser(description='PyTorch MNIST Example') +parser.add_argument('--batch-size', type=int, default=64, metavar='N', + help='input batch size for training (default: 64)') +parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', + help='input batch size for testing (default: 1000)') +parser.add_argument('--epochs', type=int, default=10, metavar='N', + help='number of epochs to train (default: 10)') +parser.add_argument('--lr', type=float, default=0.01, metavar='LR', + help='learning rate (default: 0.01)') +parser.add_argument('--momentum', type=float, default=0, metavar='M', + help='SGD momentum (default: 0)') +parser.add_argument('--no-cuda', action='store_true', default=False, + help='disables CUDA training') +parser.add_argument('--seed', type=int, default=1, metavar='S', + help='random seed (default: 1)') +parser.add_argument('--log-interval', type=int, default=10, metavar='N', + help='how many batches to wait before logging training status') +args = parser.parse_args() +args.cuda = not args.no_cuda and torch.cuda.is_available() + +torch.manual_seed(args.seed) +if args.cuda: + torch.cuda.manual_seed(args.seed) + + +kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} +train_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=True, download=True, + transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=args.batch_size, shuffle=True, **kwargs) +test_loader = torch.utils.data.DataLoader( + datasets.MNIST('../data', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=args.batch_size, shuffle=True, **kwargs) + + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(1, 10, kernel_size=5) + self.conv2 = nn.Conv2d(10, 20, kernel_size=5) + self.conv2_drop = nn.Dropout2d() + self.fc1 = nn.Linear(320, 50) + self.fc2 = nn.Linear(50, 10) + + def forward(self, x): + x = F.relu(F.max_pool2d(self.conv1(x), 2)) + x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) + x = x.view(-1, 320) + x = F.relu(self.fc1(x)) + x = F.dropout(x, training=self.training) + x = self.fc2(x) + return F.log_softmax(x) + +model = torchmodel.MVTorchModel(Net()) + +if args.cuda: + model.cuda() + +optimizer = optim.SGD(model.parameters(), lr=args.lr * mv.workers_num(), momentum=args.momentum) + +def train(epoch): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + if batch_idx % mv.workers_num() == mv.worker_id(): + if args.cuda: + data, target = data.cuda(), target.cuda() + data, target = Variable(data), Variable(target) + optimizer.zero_grad() + output = model(data) + loss = F.nll_loss(output, target) + loss.backward() + optimizer.step() + + model.cpu() + model.mv_sync() + model.cuda() + + if (batch_idx/mv.workers_num()) % args.log_interval == 0: + print('Worker: {}\tTrain Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + mv.worker_id(), epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.data[0])) + +def test(epoch): + model.eval() + test_loss = 0 + correct = 0 + for data, target in test_loader: + if args.cuda: + data, target = data.cuda(), target.cuda() + data, target = Variable(data, volatile=True), Variable(target) + output = model(data) + test_loss += F.nll_loss(output, target).data[0] + pred = output.data.max(1)[1] # get the index of the max log-probability + correct += pred.eq(target.data).cpu().sum() + + test_loss = test_loss + test_loss /= len(test_loader) # loss function already averages over batch size + print('\nWorker: {}\tTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + mv.worker_id(), test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) + + +for epoch in range(1, args.epochs + 1): + train(epoch) + test(epoch) + +mv.shutdown() \ No newline at end of file From 648283a95f6bb9503a3243b41d5c32182a75952f Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Thu, 1 Jun 2017 04:03:14 -0700 Subject: [PATCH 07/14] update .gitignore --- .gitignore | 1 + .../python/multiverso/torch_ext/torchmodel.py | 2 - .../multiverso_python.egg-info/PKG-INFO | 42 ------------------- .../multiverso_python.egg-info/SOURCES.txt | 18 -------- .../dependency_links.txt | 1 - .../multiverso_python.egg-info/not-zip-safe | 1 - .../multiverso_python.egg-info/requires.txt | 2 - .../multiverso_python.egg-info/top_level.txt | 1 - 8 files changed, 1 insertion(+), 67 deletions(-) delete mode 100644 binding/python/multiverso_python.egg-info/PKG-INFO delete mode 100644 binding/python/multiverso_python.egg-info/SOURCES.txt delete mode 100644 binding/python/multiverso_python.egg-info/dependency_links.txt delete mode 100644 binding/python/multiverso_python.egg-info/not-zip-safe delete mode 100644 binding/python/multiverso_python.egg-info/requires.txt delete mode 100644 binding/python/multiverso_python.egg-info/top_level.txt diff --git a/.gitignore b/.gitignore index 156ffd6..5697341 100644 --- a/.gitignore +++ b/.gitignore @@ -271,3 +271,4 @@ _Pvt_Extensions # Python *.pyc *.egg +binding/python/multiverso_python.egg-info/ diff --git a/binding/python/multiverso/torch_ext/torchmodel.py b/binding/python/multiverso/torch_ext/torchmodel.py index 6fb82b5..f0587c4 100644 --- a/binding/python/multiverso/torch_ext/torchmodel.py +++ b/binding/python/multiverso/torch_ext/torchmodel.py @@ -30,8 +30,6 @@ def mv_sync(self): for mv_param, last_mv_param, param in zip(self._mv_params, self._last_mv_params, self._tmobj.parameters()): mv_param.add(last_mv_param - param.data.numpy().reshape((-1,))) - mv.barrier() - for mv_param, last_mv_param, param in zip(self._mv_params, self._last_mv_params, self._tmobj.parameters()): last_mv_param = mv_param.get() param=Variable(torch.from_numpy(last_mv_param.reshape(param.data.numpy().shape))) diff --git a/binding/python/multiverso_python.egg-info/PKG-INFO b/binding/python/multiverso_python.egg-info/PKG-INFO deleted file mode 100644 index 45a36bb..0000000 --- a/binding/python/multiverso_python.egg-info/PKG-INFO +++ /dev/null @@ -1,42 +0,0 @@ -Metadata-Version: 1.1 -Name: multiverso-python -Version: 0.0.1 -Summary: Multiverso is a parameter server framework for distributed machine learning. This package can leverage multiple machines and GPUs to speed up the python programs. -Home-page: https://github.com/Microsoft/multiverso -Author: Microsoft -Author-email: UNKNOWN -License: MIT -Description: # Multiverso Python/Theano/Lasagne Binding - - - ## Introduction - Multiverso is a parameter server framework for distributed machine learning. This package can leverage multiple machines and GPUs to speed up the python programs. - - - ## Installation - - 1. (For GPU support only) Install CUDA, cuDNN according to this [guide](https://github.com/Microsoft/fb.resnet.torch/blob/multiverso/INSTALL.md). You just need finish the steps before [Install Torch](https://github.com/Microsoft/fb.resnet.torch/blob/multiverso/INSTALL.md#install-torch). - 1. Install the multiverso - * On linux: Please follow the [README](https://github.com/Microsoft/multiverso/blob/master/README.md#build) to build and install multiverso. - * On windows: You need MSBuild.exe installed and make sure your system can find it in the $PATH. Then you should run [build_dll.bat](https://github.com/Microsoft/multiverso/blob/master/src/build_dll.bat) to build the .dll file and install the .dll. There isn't auto-installer for windows now, so you have to copy the .dll to either system $PATH or the multiverso package folder. - 1. Install the requirements - * `gfortran` is required by scipy. e.g. you can install it by `sudo apt-get install gfortran` on ubuntu. - * (Optional) You need python-nose to run the unit tests. e.g. you can install it by `sudo apt-get install python-nose` on ubuntu. - 1. Install python binding with the command `sudo python setup.py install` - - - ## Run Unit Tests - ``` - nosetests - ``` - - - ## Documentation - * [Tutorial](https://github.com/Microsoft/multiverso/wiki/How-to-write-python-code-with-multiverso) - * Api documents are written as docstrings in the python source code. - * [Benchmark](https://github.com/Microsoft/multiverso/wiki/Multiverso-Python-Binding-Benchmark) - -Platform: UNKNOWN -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Science/Research -Classifier: Programming Language :: Python :: 2 diff --git a/binding/python/multiverso_python.egg-info/SOURCES.txt b/binding/python/multiverso_python.egg-info/SOURCES.txt deleted file mode 100644 index 10ee279..0000000 --- a/binding/python/multiverso_python.egg-info/SOURCES.txt +++ /dev/null @@ -1,18 +0,0 @@ -setup.py -multiverso/__init__.py -multiverso/api.py -multiverso/tables.py -multiverso/utils.py -multiverso/theano_ext/__init__.py -multiverso/theano_ext/param_manager.py -multiverso/theano_ext/sharedvar.py -multiverso/theano_ext/lasagne_ext/__init__.py -multiverso/theano_ext/lasagne_ext/param_manager.py -multiverso/torch_ext/__init__.py -multiverso/torch_ext/torchmodel.py -multiverso_python.egg-info/PKG-INFO -multiverso_python.egg-info/SOURCES.txt -multiverso_python.egg-info/dependency_links.txt -multiverso_python.egg-info/not-zip-safe -multiverso_python.egg-info/requires.txt -multiverso_python.egg-info/top_level.txt \ No newline at end of file diff --git a/binding/python/multiverso_python.egg-info/dependency_links.txt b/binding/python/multiverso_python.egg-info/dependency_links.txt deleted file mode 100644 index 250ad83..0000000 --- a/binding/python/multiverso_python.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ -https://github.com/Lasagne/Lasagne/tarball/master#egg=lasagne-0.2.dev1 diff --git a/binding/python/multiverso_python.egg-info/not-zip-safe b/binding/python/multiverso_python.egg-info/not-zip-safe deleted file mode 100644 index 8b13789..0000000 --- a/binding/python/multiverso_python.egg-info/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/binding/python/multiverso_python.egg-info/requires.txt b/binding/python/multiverso_python.egg-info/requires.txt deleted file mode 100644 index 28024b0..0000000 --- a/binding/python/multiverso_python.egg-info/requires.txt +++ /dev/null @@ -1,2 +0,0 @@ -theano>=0.8.2 -lasagne>=0.2.dev1 diff --git a/binding/python/multiverso_python.egg-info/top_level.txt b/binding/python/multiverso_python.egg-info/top_level.txt deleted file mode 100644 index fe0aaae..0000000 --- a/binding/python/multiverso_python.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -multiverso From 5f3c4ada87014c07a192669765b1c6edb676d08e Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Thu, 1 Jun 2017 04:06:36 -0700 Subject: [PATCH 08/14] update submodule --- .gitmodules | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitmodules b/.gitmodules index a8e2e14..3961dd7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "include/multiverso/updater/dcasgd"] path = include/multiverso/updater/dcasgd url = https://github.com/zhengsx/Delayed-Compensation-Asynchronous-Stochastic-Gradient-Descent-for-Multiverso.git +[submodule "include/multiverso/updater/Delayed-Compensation-Asynchronous-Stochastic-Gradient-Descent-for-Multiverso"] + path = include/multiverso/updater/Delayed-Compensation-Asynchronous-Stochastic-Gradient-Descent-for-Multiverso + url = https://github.com/Microsoft/Delayed-Compensation-Asynchronous-Stochastic-Gradient-Descent-for-Multiverso.git From 6b9011d942f7ba3e3812483a746081b45809ec79 Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Thu, 1 Jun 2017 04:09:42 -0700 Subject: [PATCH 09/14] update gitmodules --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 3961dd7..8cb7c11 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "include/multiverso/updater/dcasgd"] path = include/multiverso/updater/dcasgd - url = https://github.com/zhengsx/Delayed-Compensation-Asynchronous-Stochastic-Gradient-Descent-for-Multiverso.git -[submodule "include/multiverso/updater/Delayed-Compensation-Asynchronous-Stochastic-Gradient-Descent-for-Multiverso"] - path = include/multiverso/updater/Delayed-Compensation-Asynchronous-Stochastic-Gradient-Descent-for-Multiverso url = https://github.com/Microsoft/Delayed-Compensation-Asynchronous-Stochastic-Gradient-Descent-for-Multiverso.git From 5cd84f0bcae16f2cfb0b9de4bb77af4a6421ea9b Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Mon, 5 Jun 2017 08:58:24 -0700 Subject: [PATCH 10/14] fix bug --- binding/python/multiverso/torch_ext/torchmodel.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/binding/python/multiverso/torch_ext/torchmodel.py b/binding/python/multiverso/torch_ext/torchmodel.py index f0587c4..87f9aaf 100644 --- a/binding/python/multiverso/torch_ext/torchmodel.py +++ b/binding/python/multiverso/torch_ext/torchmodel.py @@ -24,15 +24,15 @@ def __init__(self, tmobj): for mv_param in self._mv_params: self._last_mv_params.append(mv_param.get()) for param, last_mv_param in zip(self._tmobj.parameters(), self._last_mv_params): - param=Variable(torch.from_numpy(last_mv_param.reshape(param.data.numpy().shape))) + param.data=torch.from_numpy(last_mv_param.reshape(param.data.numpy().shape)) def mv_sync(self): for mv_param, last_mv_param, param in zip(self._mv_params, self._last_mv_params, self._tmobj.parameters()): - mv_param.add(last_mv_param - param.data.numpy().reshape((-1,))) + mv_param.add( param.data.numpy().reshape((-1,)) - last_mv_param) - for mv_param, last_mv_param, param in zip(self._mv_params, self._last_mv_params, self._tmobj.parameters()): - last_mv_param = mv_param.get() - param=Variable(torch.from_numpy(last_mv_param.reshape(param.data.numpy().shape))) + for i, (mv_param, last_mv_param, param) in enumerate(zip(self._mv_params, self._last_mv_params, self._tmobj.parameters())): + self._last_mv_params[i]=mv_param.get() + param.data=torch.from_numpy(self._last_mv_params[i].reshape(param.data.numpy().shape)) def __call__(self, *args, **kwargs): return self._tmobj(*args, **kwargs) From e71203e635da1cd22f5d82692c7f87440f052b5e Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Wed, 7 Jun 2017 05:31:59 -0700 Subject: [PATCH 11/14] fix bugs and support updater --- include/multiverso/updater/sgd_updater.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/multiverso/updater/sgd_updater.h b/include/multiverso/updater/sgd_updater.h index 922165d..fa0edc8 100644 --- a/include/multiverso/updater/sgd_updater.h +++ b/include/multiverso/updater/sgd_updater.h @@ -9,7 +9,7 @@ template class SGDUpdater : public Updater { public: explicit SGDUpdater(size_t){ - Log::Debug("[SGDUpdater] Init. \n"); + Log::Debug("[SGDUpdater] Init. \n"); } void Update(size_t num_element, T* data, T* delta, AddOption*, size_t offset) override { @@ -28,4 +28,4 @@ class SGDUpdater : public Updater { } -#endif // MULTIVERSO_UPDATER_ASGD_UPDATER_H_ \ No newline at end of file +#endif // MULTIVERSO_UPDATER_ASGD_UPDATER_H_ From e0e494cd7146424ae08c96a6fa4c49d46dd7a4d7 Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Wed, 7 Jun 2017 05:36:22 -0700 Subject: [PATCH 12/14] update --- CMakeLists.txt | 6 ++++- binding/python/multiverso/api.py | 5 +++-- binding/python/multiverso/tables.py | 10 ++++++--- .../python/multiverso/torch_ext/torchmodel.py | 4 ++-- include/multiverso/c_api.h | 3 +++ src/c_api.cpp | 22 +++++++++++++++++++ src/updater/updater.cpp | 6 ++--- 7 files changed, 44 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cdcf9bb..88227d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,10 @@ if(USE_HDFS) LINK_DIRECTORIES(${JVM_LIB}) endif(USE_HDFS) +if(ENABLE_DCASGD) + add_definitions(-DENABLE_DCASGD) +endif(ENABLE_DCASGD) + include_directories(${PROJECT_SOURCE_DIR}/include) set(MULTIVERSO_DIR ${PROJECT_SOURCE_DIR}) @@ -43,4 +47,4 @@ configure_file( add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) -add_definitions(-DENABLE_DCASGD) + diff --git a/binding/python/multiverso/api.py b/binding/python/multiverso/api.py index 1a96177..0fd8081 100644 --- a/binding/python/multiverso/api.py +++ b/binding/python/multiverso/api.py @@ -9,7 +9,7 @@ mv_lib = Loader.get_lib() -def init(sync=False, updater='sgd'): +def init(sync=False, updater=None): '''Initialize mutliverso. This should be called only once before training at the beginning of the @@ -29,7 +29,8 @@ def init(sync=False, updater='sgd'): args = [b""] # the first argument will be ignored. So we put a placeholder here if sync: args.append(b"-sync=true") - args.append(b"-updater=sgd") + if updater: + args.append(b"-updater_type="+updater) n = len(args) args_type = ctypes.c_char_p * n mv_lib.MV_Init(ctypes.pointer(ctypes.c_int(n)), args_type(*[ctypes.c_char_p(arg) for arg in args])) diff --git a/binding/python/multiverso/tables.py b/binding/python/multiverso/tables.py index 4ec312b..757a40f 100644 --- a/binding/python/multiverso/tables.py +++ b/binding/python/multiverso/tables.py @@ -65,7 +65,7 @@ def get(self): mv_lib.MV_GetArrayTable(self._handler, data.ctypes.data_as(C_FLOAT_P), self._size) return data - def add(self, data, sync=False): + def add(self, data, sync=False, lr=0.1, mom=0.0, rho=0.0, lam=0.0): '''add the data to the multiverso ArrayTable Data type of `data` is numpy.ndarray with one-dimensional @@ -76,9 +76,13 @@ def add(self, data, sync=False): data = convert_data(data) assert(data.size == self._size) if sync: - mv_lib.MV_AddArrayTable(self._handler, data.ctypes.data_as(C_FLOAT_P), self._size) + # mv_lib.MV_AddArrayTable(self._handler, data.ctypes.data_as(C_FLOAT_P), self._size) + mv_lib.MV_AddArrayTableOption(self._handler, data.ctypes.data_as(C_FLOAT_P), self._size, + ctypes.c_float(lr), ctypes.c_float(mom), ctypes.c_float(rho), ctypes.c_float(lam)) else: - mv_lib.MV_AddAsyncArrayTable(self._handler, data.ctypes.data_as(C_FLOAT_P), self._size) + # mv_lib.MV_AddAsyncArrayTable(self._handler, data.ctypes.data_as(C_FLOAT_P), self._size) + mv_lib.MV_AddAsyncArrayTableOption(self._handler, data.ctypes.data_as(C_FLOAT_P), self._size, + ctypes.c_float(lr), ctypes.c_float(mom), ctypes.c_float(rho), ctypes.c_float(lam)) class MatrixTableHandler(TableHandler): diff --git a/binding/python/multiverso/torch_ext/torchmodel.py b/binding/python/multiverso/torch_ext/torchmodel.py index 87f9aaf..b169458 100644 --- a/binding/python/multiverso/torch_ext/torchmodel.py +++ b/binding/python/multiverso/torch_ext/torchmodel.py @@ -26,9 +26,9 @@ def __init__(self, tmobj): for param, last_mv_param in zip(self._tmobj.parameters(), self._last_mv_params): param.data=torch.from_numpy(last_mv_param.reshape(param.data.numpy().shape)) - def mv_sync(self): + def mv_sync(self, lr=0.1, mom=0.0, rho=0.0, lam=0.0): for mv_param, last_mv_param, param in zip(self._mv_params, self._last_mv_params, self._tmobj.parameters()): - mv_param.add( param.data.numpy().reshape((-1,)) - last_mv_param) + mv_param.add(last_mv_param - param.data.numpy().reshape((-1,)), lr=lr, mom=mom, rho=rho, lam=lam) for i, (mv_param, last_mv_param, param) in enumerate(zip(self._mv_params, self._last_mv_params, self._tmobj.parameters())): self._last_mv_params[i]=mv_param.get() diff --git a/include/multiverso/c_api.h b/include/multiverso/c_api.h index 3a566b6..d9abb9b 100644 --- a/include/multiverso/c_api.h +++ b/include/multiverso/c_api.h @@ -32,8 +32,11 @@ DllExport void MV_GetArrayTable(TableHandler handler, float* data, int size); DllExport void MV_AddArrayTable(TableHandler handler, float* data, int size); +DllExport void MV_AddArrayTableOption(TableHandler handler, float* data, int size, float lr, float mom, float rho, float lambda); + DllExport void MV_AddAsyncArrayTable(TableHandler handler, float* data, int size); +DllExport void MV_AddAsyncArrayTableOption(TableHandler handler, float* data, int size, float lr, float mom, float rho, float lambda); // Matrix Table DllExport void MV_NewMatrixTable(int num_row, int num_col, TableHandler* out); diff --git a/src/c_api.cpp b/src/c_api.cpp index a952ff1..56122b6 100644 --- a/src/c_api.cpp +++ b/src/c_api.cpp @@ -4,6 +4,7 @@ #include "multiverso/table/array_table.h" #include "multiverso/table/matrix_table.h" #include "multiverso/util/log.h" +#include "multiverso/updater/updater.h" extern "C" { @@ -46,11 +47,32 @@ void MV_AddArrayTable(TableHandler handler, float* data, int size) { worker->Add(data, size); } +void MV_AddArrayTableOption(TableHandler handler, float* data, int size, float lr, float mom, float rho, float lambda) { + auto worker = reinterpret_cast*>(handler); + multiverso::AddOption option; + option.set_worker_id(multiverso::MV_WorkerId()); + option.set_learning_rate(lr); + option.set_momentum(mom); + option.set_rho(rho); + option.set_lambda(lambda); + worker->Add(data, size, &option); +} + void MV_AddAsyncArrayTable(TableHandler handler, float* data, int size) { auto worker = reinterpret_cast*>(handler); worker->AddAsync(data, size); } +void MV_AddAsyncArrayTableOption(TableHandler handler, float* data, int size, float lr, float mom, float rho, float lambda) { + auto worker = reinterpret_cast*>(handler); + multiverso::AddOption option; + option.set_worker_id(multiverso::MV_WorkerId()); + option.set_learning_rate(lr); + option.set_momentum(mom); + option.set_rho(rho); + option.set_lambda(lambda); + worker->AddAsync(data, size, &option); +} // MatrixTable void MV_NewMatrixTable(int num_row, int num_col, TableHandler* out) { diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index bfd7d66..3f847ba 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -16,9 +16,6 @@ namespace multiverso { MV_DEFINE_string(updater_type, "default", "multiverso server updater type"); MV_DEFINE_int(omp_threads, 4 , "#theads used by openMP for updater"); -#ifdef ENABLE_DCASGD -MV_DEFINE_bool(is_pipelined, false, "Only used for CNTK - DCASGD"); -#endif template void Updater::Update(size_t num_element, T* data, T* delta, @@ -51,9 +48,10 @@ Updater* Updater::GetUpdater(size_t size) { if (type == "adagrad") return new AdaGradUpdater(size); if (type == "momentum_sgd") return new MomentumUpdater(size); #ifdef ENABLE_DCASGD - if (type == "dcasgd") return new DCASGDUpdater(size, MV_CONFIG_is_pipelined); + if (type == "dcasgd") return new DCASGDUpdater(size); #endif // Default: simple updater + Log::Info("[Updater] Init. \n"); return new Updater(); } From d46cd3d692a0d3260b93dffb7a97088ea91edb90 Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Wed, 7 Jun 2017 05:43:33 -0700 Subject: [PATCH 13/14] update submodule --- include/multiverso/updater/dcasgd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/multiverso/updater/dcasgd b/include/multiverso/updater/dcasgd index 4bdd2c3..ab8dab6 160000 --- a/include/multiverso/updater/dcasgd +++ b/include/multiverso/updater/dcasgd @@ -1 +1 @@ -Subproject commit 4bdd2c38702be6f443f9540f9d76d5190cf13e06 +Subproject commit ab8dab629c725cdf322534098cc838c6a0aa86e1 From 96d36c78a37b6a75b0d374eaa2a7af2bfc0a2b07 Mon Sep 17 00:00:00 2001 From: "Shuxin Zheng (MSR Student-Person Consulting)" Date: Wed, 7 Jun 2017 23:44:37 -0700 Subject: [PATCH 14/14] add getstate func --- binding/python/multiverso/torch_ext/torchmodel.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/binding/python/multiverso/torch_ext/torchmodel.py b/binding/python/multiverso/torch_ext/torchmodel.py index b169458..23ef007 100644 --- a/binding/python/multiverso/torch_ext/torchmodel.py +++ b/binding/python/multiverso/torch_ext/torchmodel.py @@ -37,10 +37,15 @@ def mv_sync(self, lr=0.1, mom=0.0, rho=0.0, lam=0.0): def __call__(self, *args, **kwargs): return self._tmobj(*args, **kwargs) + def __getstate__(self): + odict = self.__dict__.copy() + del odict['_mv_params'] + return odict + def __getattribute__(self, attr): if attr in ['_tmobj', '_mv_params', '_last_mv_params']: return object.__getattribute__(self, attr) - elif attr in ['mv_sync', '__call__']: + elif attr in ['mv_sync', '__call__','__getstate__']: return getattr(MVTorchModel, attr).__get__(self) else: return getattr(self._tmobj, attr)