dmlc · immars · Mar 16, 2015 · Mar 16, 2015 · Mar 18, 2015 · Mar 19, 2015
diff --git a/.gitignore b/.gitignore
@@ -11,3 +11,4 @@
 *.pb.cc
 .*
 /script/van*
+Makefile.config
diff --git a/Makefile b/Makefile
@@ -1,21 +1,26 @@
+
+CONFIG_FILE := Makefile.config
+include $(CONFIG_FILE)
+
 CC = g++
 
-# OPT = -O0 -ggdb
+# OPT = -O0 -ggdb -DDEBUG
 OPT = -O3 -ggdb
 
-THIRD_PATH=$(shell pwd)/third_party
+THIRD_PATH=$(shell pwd -L )/third_party
+
 STATIC_THIRD_LIB=0
 ifeq ($(STATIC_THIRD_LIB), 1)
 THIRD_LIB=$(addprefix $(THIRD_PATH)/lib/, libgflags.a libzmq.a libprotobuf.a libglog.a libz.a  libsnappy.a)
 else
-THIRD_LIB=-L$(THIRD_PATH)/lib -lgflags -lzmq -lprotobuf -lglog -lz -lsnappy
+THIRD_LIB=-L$(THIRD_PATH)/lib -lgflags -lzmq -lprotobuf -lglog -lz -lsnappy -L$(CUDA_PATH)/lib64 -lcudart
 endif
 # THIRD_LIB+=-ltcmalloc_and_profiler
 
 WARN = -Wall -Wno-unused-function -finline-functions -Wno-sign-compare #-Wconversion
-INCPATH = -I./src -I$(THIRD_PATH)/include
+INCPATH = -I./src -I$(THIRD_PATH)/include -I/usr/include/eigen3 -I$(CAFFE_PATH)/include -I$(CAFFE_PATH)/build/src  -I$(CUDA_PATH)/include
 CFLAGS = -std=c++0x $(WARN) $(OPT) $(INCPATH)
-LDFLAGS += $(THIRD_LIB) -lpthread -lrt
+LDFLAGS += $(THIRD_LIB) -lboost_thread -lboost_system -lpthread -lrt -lcaffe -L$(CAFFE_PATH)/build/lib -Wl,-rpath=$(CAFFE_PATH)/build/lib -Wl,-rpath=$(THIRD_PATH)/lib
 
 PS_LIB = build/libps.a
 PS_MAIN = build/libpsmain.a
@@ -30,6 +35,20 @@ app: build/ps
 build/hello: build/app/hello_world/main.o $(PS_LIB) $(PS_MAIN)
 	$(CC) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
+build/caffe: build/app/caffe/caffe_main.o $(PS_LIB)
+	$(CC) $(CFLAGS) $^ $(LDFLAGS) -o $@	
+
+build/caffe_sync: build/app/caffe/caffe_synced.o $(PS_LIB)
+	$(CC) $(CFLAGS) $^ $(LDFLAGS) -o $@	
+
+build/caffe_share: build/app/caffe/caffe_share.o $(PS_LIB)
+	$(CC) $(CFLAGS) $^ $(LDFLAGS) -o $@	
+
+build/caffe_async_share: build/app/caffe/caffe_async_share.o $(PS_LIB)
+	$(CC) $(CFLAGS) $^ $(LDFLAGS) -o $@	
+
+caffe_all: build/caffe build/caffe_sync build/caffe_share build/caffe_async_share
+
 sys_srcs	= $(wildcard src/util/*.cc) $(wildcard src/data/*.cc) \
 			  $(wildcard src/system/*.cc) $(wildcard src/filter/*.cc)
 sys_protos	= $(wildcard src/*/proto/*.proto)

diff --git a/Makefile.config.example b/Makefile.config.example
@@ -0,0 +1,8 @@
+##
+
+# CUDA path for include
+CUDA_PATH := /usr/local/cuda
+
+# caffe source path
+# need to build caffe successfully
+CAFFE_PATH := /data/ML/caffe/caffe
diff --git a/conf/caffe.as.conf b/conf/caffe.as.conf
@@ -0,0 +1,17 @@
+# async shared
+PS_PATH /home/immars/work/ML/distributed/parameter_server/build/caffe_async_share
+PUSH 4
+PULL 8
+SCHEDULER 192.168.1.108
+
+SERVER 192.168.1.108 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML solver.maxmap.prototxt 0 snapshot/bvlc_maxmap_iter_1210000.solverstate
+# SERVER 192.168.1.108 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML solver.maxmap.prototxt 0
+WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML solver.maxmap.prototxt 0 W0,W1,W2,W3
+# WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W1 solver.maxmap.prototxt 1
+# WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W2 solver.maxmap.prototxt 2
+# WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W3 solver.maxmap.prototxt 3
+WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML solver.maxmap.prototxt 0 W0,W1,W2,W3
+# WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W1 solver.maxmap.prototxt 1
+# WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W2 solver.maxmap.prototxt 2
+# WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W3 solver.maxmap.prototxt 3
+
diff --git a/conf/caffe.conf b/conf/caffe.conf
@@ -0,0 +1,6 @@
+PS_PATH /home/immars/work/ML/distributed/parameter_server/build/caffe
+SCHEDULER 192.168.1.108
+
+SERVER 192.168.1.108 /home/immars/work/ML/caffe/caffe/models/bvlc_reference_caffenet/S0 solver.prototxt -1
+WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_reference_caffenet/W0 solver.prototxt 0
+WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_reference_caffenet/W1 solver.prototxt 1
diff --git a/conf/caffe.googlenet.conf b/conf/caffe.googlenet.conf
@@ -0,0 +1,16 @@
+PS_PATH /home/immars/work/ML/distributed/parameter_server/build/caffe_sync
+PUSH 8
+PULL 8
+SCHEDULER 192.168.1.108
+
+# SERVER 192.168.1.108 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML solver.adadelta.prototxt -1 snapshot/bvlc_googlenet_iter_220000.solverstate
+SERVER 192.168.1.108 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML solver.adadelta.s.prototxt -1 snapshot/bvlc_googlenet_iter_140000.solverstate
+WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W0 solver.adadelta.c.prototxt 0
+WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W1 solver.adadelta.c.prototxt 1
+WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W2 solver.adadelta.c.prototxt 2
+WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W3 solver.adadelta.c.prototxt 3
+WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W0 solver.adadelta.c.prototxt 0
+WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W1 solver.adadelta.c.prototxt 1
+WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W2 solver.adadelta.c.prototxt 2
+WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W3 solver.adadelta.c.prototxt 3
+
diff --git a/conf/caffe.maxmap.conf b/conf/caffe.maxmap.conf
@@ -0,0 +1,15 @@
+PS_PATH /home/immars/work/ML/distributed/parameter_server/build/caffe_sync
+PUSH 8
+PULL 8
+SCHEDULER 192.168.1.108
+
+SERVER 192.168.1.108 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML solver.maxmap.prototxt -1
+WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W0 solver.maxmap.prototxt 0
+# WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W1 solver.maxmap.prototxt 1
+# WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W2 solver.maxmap.prototxt 2
+# WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W3 solver.maxmap.prototxt 3
+# WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W0 solver.maxmap.prototxt 0
+# WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W1 solver.maxmap.prototxt 1
+# WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W2 solver.maxmap.prototxt 2
+# WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W3 solver.maxmap.prototxt 3
+
diff --git a/conf/caffe.share.conf b/conf/caffe.share.conf
@@ -0,0 +1,15 @@
+PS_PATH /home/immars/work/ML/distributed/parameter_server/build/caffe_share
+PUSH 8
+PULL 8
+SCHEDULER 192.168.1.108
+
+SERVER 192.168.1.108 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML solver.maxmap.prototxt -1
+WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML solver.maxmap.prototxt -1 W0,W1,W2,W3
+# WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W1 solver.maxmap.prototxt 1
+# WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W2 solver.maxmap.prototxt 2
+# WORKER 192.168.1.110 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W3 solver.maxmap.prototxt 3
+WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML solver.maxmap.prototxt 0 W0,W1,W2,W3
+# WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W1 solver.maxmap.prototxt 1
+# WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W2 solver.maxmap.prototxt 2
+# WORKER 192.168.1.112 /home/immars/work/ML/caffe/caffe/models/bvlc_googlenet/ML/W3 solver.maxmap.prototxt 3
+
diff --git a/script/caffe_kill_lan.sh b/script/caffe_kill_lan.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# set -x
+# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../third_party/lib
+
+if [ $# -lt 1 ]; then
+    echo "usage: ./caffe_kill_lan.sh conf_path"
+    echo "solver.prototxt resides in subdirectories[S0,S1,...,W0,W1,...] of root_dir"
+    exit -1;
+fi
+
+conf=$1
+tmp=$( mktemp )
+grep -v ^# $conf > $tmp
+conf=$tmp
+app=$( grep PS_PATH $conf | awk -F'/' '{print $NF;}' )
+echo "app: $app"
+echo "kill local"
+
+killall caffe || killall $app
+
+echo "kill servers"
+# kill servers
+grep -E "WORKER|SERVER" $conf | awk '{print $2;}' | sort | uniq | awk  -v q="'" -v app="$app" '
+{
+    cmd="ssh immars@" $0 " \"killall caffe || killall " app " \" ";
+    print cmd;
+    system(cmd);
+}
+'
diff --git a/script/caffe_lan.sh b/script/caffe_lan.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+# set -x
+# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../third_party/lib
+if [ $# -lt 1 ]; then
+    echo "usage: ./local.sh num_servers num_workers root_dir solver.prototxt [args..]"
+    echo "solver.prototxt resides in subdirectories[S0,S1,...,W0,W1,...] of root_dir"
+    exit -1;
+fi
+
+conf=$1
+tmp=$( mktemp )
+grep -v ^# $conf > $tmp
+conf=$tmp
+
+bin=$( grep PS_PATH $conf | awk '{print $2;}' )
+sch_ip=$( grep SCHEDULER $conf | awk '{print $2;}' )
+num_servers=$( grep SERVER $conf | wc -l )
+num_workers=$( grep WORKER $conf | wc -l )
+pullstep=$( grep PULL $conf | awk '{print $2;}' )
+pushstep=$( grep PUSH $conf | awk '{print $2;}' )
+
+arg="-num_servers ${num_servers} -num_workers ${num_workers} $@" #" -app ${dir}/$@"
+
+echo "$bin $conf $sch_ip $num_servers $num_workers"
+killall -q caffe
+
+silence=">/dev/null 2>/dev/null"
+
+
+# start the scheduler
+Sch="role:SCHEDULER,hostname:'$sch_ip',port:8001,id:'H'"
+${bin} -my_node ${Sch} -scheduler ${Sch} ${arg} >/dev/null 2>/dev/null &
+
+# start servers
+grep SERVER $conf | awk -v bin=$bin -v sch=$Sch -v nums=$num_servers -v numw=$num_workers -vpullstep=$pullstep -vpushstep=$pushstep -v q="'" '
+BEGIN{port=9600;id=0;}
+{
+    ip=$2;wd=$3;solver=$4;gpu=$5;snapshot=$6;
+    if(""!=snapshot){
+        snapshot= " --snapshot=" snapshot;
+    }
+    cmd="ssh -f -n immars@" ip " \"source /etc/profile && cd " wd " && nohup " bin " -num_servers " nums " -num_workers " numw " -my_node \\\"role:SERVER,hostname:" q ip q ",port:" port ",id:" q "S" id q "\\\" -scheduler \\\"" sch "\\\" --solver=" solver " --pullstep=" pullstep " --pushstep=" pushstep " --gpu=" gpu " " snapshot " >" wd "/stdout.txt 2>&1 < /dev/null &\" ";
+    print cmd;
+    system(cmd);
+    port=port+1;id=id+1;
+}
+'
+
+grep WORKER $conf | awk -v bin=$bin -v sch=$Sch -v nums=$num_servers -v numw=$num_workers -vpullstep=$pullstep -vpushstep=$pushstep -v q="'" '
+BEGIN{port=9500;id=0;}
+{
+    ip=$2;wd=$3;solver=$4;gpu=$5;workers=$6;
+    if(""!=workers){
+        workers = " --workers=" workers;
+    }
+    cmd="ssh -f -n immars@" ip " \"source /etc/profile && cd " wd " && nohup " bin " -num_servers " nums " -num_workers " numw " -my_node \\\"role:WORKER,hostname:" q ip q ",port:" port ",id:" q "W" id q "\\\" -scheduler \\\"" sch "\\\" --solver=" solver " --pullstep=" pullstep " --pushstep=" pushstep " --synced=true --gpu=" gpu " " workers " >" wd "/stdout.txt 2>&1 < /dev/null &\" ";
+    print cmd;
+    system(cmd);
+    port=port+1;id=id+1;
+}
+'
+
diff --git a/script/caffe_local.sh b/script/caffe_local.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+# set -x
+# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../third_party/lib
+if [ $# -lt 3 ]; then
+    echo "usage: ./local.sh num_servers num_workers root_dir solver.prototxt [args..]"
+    echo "solver.prototxt resides in subdirectories[S0,S1,...,W0,W1,...] of root_dir"
+    exit -1;
+fi
+
+bin=$( pwd -L )/build/caffe_sync
+num_servers=$1
+shift
+num_workers=$1
+shift
+root_dir=$1
+shift
+solver=$1
+shift
+arg="-num_servers ${num_servers} -num_workers ${num_workers} $@" #" -app ${dir}/$@"
+
+
+# killall -q $(basename ${bin})
+killall -q ${bin}
+sleep 1
+
+silence=">/dev/null 2>/dev/null"
+
+
+# start the scheduler
+Sch="role:SCHEDULER,hostname:'127.0.0.1',port:8001,id:'H'"
+${bin} -my_node ${Sch} -scheduler ${Sch} ${arg}  &
+
+# start servers
+for ((i=0; i<${num_servers}; ++i)); do
+    port=$((9600 + ${i}))
+    id=S${i}
+    N="role:SERVER,hostname:'127.0.0.1',port:${port},id:'${id}'"
+    # HEAPPROFILE=/tmp/S${i} \
+    # CPUPROFILE=/tmp/S${i} \
+    echo "cd $root_dir/$id/ && ${bin} -my_node ${N} -scheduler ${Sch} --solver=$solver ${arg} >$root_dir/$id/stdout.txt 2>&1 &"
+    cd $root_dir/$id/ && ${bin} -my_node ${N} -scheduler ${Sch} --pullstep=2 --pushstep=2 --solver=$solver ${arg} >$root_dir/$id/stdout.txt 2>&1 &
+done
+
+# start workers
+for ((i=0; i<${num_workers}; ++i)); do
+    port=$((9500 + ${i}))
+    id=W${i}
+    N="role:WORKER,hostname:'127.0.0.1',port:${port},id:'${id}'"
+    # HEAPPROFILE=/tmp/W${i} \
+    # CPUPROFILE=/tmp/W${i} \
+    cd $root_dir/$id/ && ${bin} -my_node ${N} -scheduler ${Sch} --pullstep=2 --pushstep=2 --solver=$solver ${arg} >$root_dir/$id/stdout.txt 2>&1 &
+done
+
+wait
-Original file line number
+Diff line change
@@ Expand Up / @@ -11,3 +11,4 @@ @@
     *.pb.cc
     .*
     /script/van*
+    Makefile.config