Merge changes to main repo

.gitignore

@@ -34,5 +34,5 @@ output/
 #
 ContainerFiles/ssh
 id_rsa
-
+id_rsa.pub
 

ContainerFiles/dev-env.nix

@@ -14,6 +14,7 @@ with import <nixpkgs> { };
     #
     nix
     bash
+    vim
 
     #
     # MPI-related packages
@@ -30,4 +31,4 @@ with import <nixpkgs> { };
 #
 # Refs:
 # https://stackoverflow.com/questions/46165918/how-to-get-the-name-from-a-nixpkgs-derivation-in-a-nix-expression-to-be-used-by/46173041#46173041
-##
+##

ContainerFiles/env_backup.drv

@@ -1 +1 @@
-Derive([("out","/nix/store/njmwm90cmzk9mx3pvfv5g0dlqg5yfgaj-openmpi-dev-env","","")],[("/nix/store/10yajwbqfbc1qjsf1k3m6ycpcpvykf3g-nix-2.1.3.drv",["man","out"]),("/nix/store/3xynsfmdjz4ifc27292cxgsnkpqb089a-gfortran-wrapper-7.3.0.drv",["out"]),("/nix/store/6s8iax7fp6j6813nr8kbcjclb18zf7ji-binutils-wrapper-2.30.drv",["out"]),("/nix/store/sv2snvfj65dqh9md4fn1sbin3qx9n7dx-openssh-7.9p1.drv",["out"]),("/nix/store/vsmrpdsq4rmdzbrhhzdf9vffwf4p2sfg-bash-4.4-p23.drv",["out"]),("/nix/store/w4254sxdxdgsbykfzmp6g31pjs32yz1q-perl-5.28.0.drv",["out"]),("/nix/store/x30k04wzg72v9da7zd0ymna05zbzdvlc-stdenv-linux.drv",["out"]),("/nix/store/yxh1vdrhmvv9z9rmkhpdq5vla9gp6bjn-openmpi-3.1.3.drv",["out"]),("/nix/store/zmkbca9m7k4v5k7qvq1s1nc1vjsbsqmf-git-2.19.1.drv",["out"])],["/nix/store/9g4wc31j7a2xp22xpgwr0qssfxahxdzl-builder.pl","/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"],"x86_64-linux","/nix/store/n2hjbpkf4c0m48945ivxs3lmsczzw2rg-bash-4.4-p23/bin/bash",["-e","/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"],[("buildCommand","/nix/store/1dzsd6drn31dgp3hk74x2h87j6jdj55b-perl-5.28.0/bin/perl -w /nix/store/9g4wc31j7a2xp22xpgwr0qssfxahxdzl-builder.pl\neval \"$postBuild\"\n"),("buildInputs",""),("builder","/nix/store/n2hjbpkf4c0m48945ivxs3lmsczzw2rg-bash-4.4-p23/bin/bash"),("checkCollisionContents","1"),("configureFlags",""),("depsBuildBuild",""),("depsBuildBuildPropagated",""),("depsBuildTarget",""),("depsBuildTargetPropagated",""),("depsHostHost",""),("depsHostHostPropagated",""),("depsTargetTarget",""),("depsTargetTargetPropagated",""),("doCheck",""),("doInstallCheck",""),("extraPrefix",""),("ignoreCollisions",""),("manifest",""),("name","openmpi-dev-env"),("nativeBuildInputs",""),("out","/nix/store/njmwm90cmzk9mx3pvfv5g0dlqg5yfgaj-openmpi-dev-env"),("outputs","out"),("pathsToLink","/"),("pkgs","[{\"paths\":[\"/nix/store/liakj490yfk7rl93744dbf97iql4y7sr-git-2.19.1\"],\"priority\":5},{\"paths\":[\"/nix/store/3yb7q19q99s6dvvriz4r6rxrk61imicz-nix-2.1.3\",\"/nix/store/vbggwyzxqvbvr2n7j1mjxpb8k3sqsg7p-nix-2.1.3-man\"],\"priority\":5},{\"paths\":[\"/nix/store/n2hjbpkf4c0m48945ivxs3lmsczzw2rg-bash-4.4-p23\"],\"priority\":10},{\"paths\":[\"/nix/store/lr095ngxfchl95d97xrd6yyn1kgcf72c-binutils-wrapper-2.30\"],\"priority\":5},{\"paths\":[\"/nix/store/yrnlqvkig7gwpfp30h199f15ifjp93d2-gfortran-wrapper-7.3.0\"],\"priority\":5},{\"paths\":[\"/nix/store/2qdbh4isw5va55728rmwyqm8hwj0ipz8-openmpi-3.1.3\"],\"priority\":5},{\"paths\":[\"/nix/store/xwrxp7slvc3xaprzm4c5pk8qmq2hnnhm-openssh-7.9p1\"],\"priority\":5}]"),("postBuild",""),("preferLocalBuild","1"),("propagatedBuildInputs",""),("propagatedNativeBuildInputs",""),("stdenv","/nix/store/iqjxz59ijw8nsnh51qxcgq00zgha18dk-stdenv-linux"),("strictDeps",""),("system","x86_64-linux")])
+Derive([("out","/nix/store/rbf7xdpq145w1nzkmf8i9qygiwr520f4-openmpi-dev-env","","")],[("/nix/store/10yajwbqfbc1qjsf1k3m6ycpcpvykf3g-nix-2.1.3.drv",["man","out"]),("/nix/store/3xynsfmdjz4ifc27292cxgsnkpqb089a-gfortran-wrapper-7.3.0.drv",["out"]),("/nix/store/6s8iax7fp6j6813nr8kbcjclb18zf7ji-binutils-wrapper-2.30.drv",["out"]),("/nix/store/i586hfrcqs2z581739skdigdz0dqc1fn-vim-8.1.0490.drv",["out"]),("/nix/store/sv2snvfj65dqh9md4fn1sbin3qx9n7dx-openssh-7.9p1.drv",["out"]),("/nix/store/vsmrpdsq4rmdzbrhhzdf9vffwf4p2sfg-bash-4.4-p23.drv",["out"]),("/nix/store/w4254sxdxdgsbykfzmp6g31pjs32yz1q-perl-5.28.0.drv",["out"]),("/nix/store/x30k04wzg72v9da7zd0ymna05zbzdvlc-stdenv-linux.drv",["out"]),("/nix/store/yxh1vdrhmvv9z9rmkhpdq5vla9gp6bjn-openmpi-3.1.3.drv",["out"]),("/nix/store/zmkbca9m7k4v5k7qvq1s1nc1vjsbsqmf-git-2.19.1.drv",["out"])],["/nix/store/9g4wc31j7a2xp22xpgwr0qssfxahxdzl-builder.pl","/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"],"x86_64-linux","/nix/store/n2hjbpkf4c0m48945ivxs3lmsczzw2rg-bash-4.4-p23/bin/bash",["-e","/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"],[("buildCommand","/nix/store/1dzsd6drn31dgp3hk74x2h87j6jdj55b-perl-5.28.0/bin/perl -w /nix/store/9g4wc31j7a2xp22xpgwr0qssfxahxdzl-builder.pl\neval \"$postBuild\"\n"),("buildInputs",""),("builder","/nix/store/n2hjbpkf4c0m48945ivxs3lmsczzw2rg-bash-4.4-p23/bin/bash"),("checkCollisionContents","1"),("configureFlags",""),("depsBuildBuild",""),("depsBuildBuildPropagated",""),("depsBuildTarget",""),("depsBuildTargetPropagated",""),("depsHostHost",""),("depsHostHostPropagated",""),("depsTargetTarget",""),("depsTargetTargetPropagated",""),("doCheck",""),("doInstallCheck",""),("extraPrefix",""),("ignoreCollisions",""),("manifest",""),("name","openmpi-dev-env"),("nativeBuildInputs",""),("out","/nix/store/rbf7xdpq145w1nzkmf8i9qygiwr520f4-openmpi-dev-env"),("outputs","out"),("pathsToLink","/"),("pkgs","[{\"paths\":[\"/nix/store/liakj490yfk7rl93744dbf97iql4y7sr-git-2.19.1\"],\"priority\":5},{\"paths\":[\"/nix/store/3yb7q19q99s6dvvriz4r6rxrk61imicz-nix-2.1.3\",\"/nix/store/vbggwyzxqvbvr2n7j1mjxpb8k3sqsg7p-nix-2.1.3-man\"],\"priority\":5},{\"paths\":[\"/nix/store/n2hjbpkf4c0m48945ivxs3lmsczzw2rg-bash-4.4-p23\"],\"priority\":10},{\"paths\":[\"/nix/store/j0prwzyw07rq2pjg32al48nxdclr0nh3-vim-8.1.0490\"],\"priority\":5},{\"paths\":[\"/nix/store/lr095ngxfchl95d97xrd6yyn1kgcf72c-binutils-wrapper-2.30\"],\"priority\":5},{\"paths\":[\"/nix/store/yrnlqvkig7gwpfp30h199f15ifjp93d2-gfortran-wrapper-7.3.0\"],\"priority\":5},{\"paths\":[\"/nix/store/2qdbh4isw5va55728rmwyqm8hwj0ipz8-openmpi-3.1.3\"],\"priority\":5},{\"paths\":[\"/nix/store/xwrxp7slvc3xaprzm4c5pk8qmq2hnnhm-openssh-7.9p1\"],\"priority\":5}]"),("postBuild",""),("preferLocalBuild","1"),("propagatedBuildInputs",""),("propagatedNativeBuildInputs",""),("stdenv","/nix/store/iqjxz59ijw8nsnh51qxcgq00zgha18dk-stdenv-linux"),("strictDeps",""),("system","x86_64-linux")])

Dockerfile

@@ -1,5 +1,6 @@
-ARG BASEOS
-FROM cornellcac/nix_${BASEOS}_base:2cd324f48c57f66d476a7894c6a957a78e49b116
+#ARG BASEOS
+#FROM cornellcac/nix_${BASEOS}_base:2cd324f48c57f66d476a7894c6a957a78e49b116
+FROM cornellcac/nix_alpine_base:2cd324f48c57f66d476a7894c6a957a78e49b116
 
 USER root
 ARG ADDUSER
@@ -20,12 +21,17 @@ RUN chown -R $nixuser:$nixuser $HOME/.ssh
 USER $nixuser
 RUN $nixenv && cd /tmp && sh $ENVSDIR/persist-env.sh $ENVSDIR/dev-env.nix
 RUN $nixenv && cd $HOME && \
+  eval $(ssh-agent) && \
+  ssh-add $HOME/.ssh/id_rsa && \
   cat /tmp/bitbucket_ssh_keyscan >> $HOME/.ssh/known_hosts && \
-  git clone https://github.com/bernardoct/WaterPaths.git && \
-  cd WaterPaths && git checkout 7b380c388c08e9891e345c59125af7b077fd218b && \
+  git clone https://github.com/federatedcloud/WaterPaths.git && \
+  cd WaterPaths && \
+  git checkout aaf3c8ea9755dab072f310ab596cce3d387af8ce && \
   nix-shell -I ssh-config-file=./nix/.ssh-standard-config -I ssh-auth-sock=$SSH_AUTH_SOCK nix/shell.nix --run "make borg" && \
   rm -f $HOME/.ssh/id_rsa
-#
+
+# Testing
+COPY nix/remora.nix $HOME/WaterPaths
 
 USER root
 
@@ -46,7 +52,11 @@ RUN SSHD_PATH=$(su -c "$nixenv && nix-build '<nixpkgs>' --no-build-output --no-o
   mkdir -p /var/empty/sshd/etc && \
   cd /var/empty/sshd/etc && \
   ln -s /etc/localtime localtime
-
+
+# make mpi and orted executable symlinks on path
+RUN for i in $(ls /nixenv/nixuser/.nix-profile/bin) ; do ln -s /nixenv/nixuser/.nix-profile/bin/"$i" /usr/bin ; done
+RUN sed -i 's|^nixuser.*|nixuser:x:1000:1000::/home/nixuser:/nixenv/nixuser/.nix-profile/bin/bash|' /etc/passwd
+
 USER $nixuser
 
 
@@ -65,6 +75,7 @@ ADD ContainerFiles/ssh/id_rsa.mpi.pub ${SSHDIR}/authorized_keys
 
 USER root
 
+RUN passwd -d nixuser
 RUN chmod -R 600 ${SSHDIR}* && \
     chown -R ${nixuser}:${nixuser} ${SSHDIR}
 

README.md

@@ -73,6 +73,7 @@ source build-docker.sh
 source docker-compose.sh up --scale mpi_head=1 --scale mpi_node=3
 docker exec -u nixuser -it waterpaths_mpi_head_1 /bin/sh
 cd ~/WaterPaths
+nix-shell nix/remora.nix
 ```
 
 Then you can run with Borg in MPI-mode, e.g.:

borgms_hybrid_openmp_mpi.sh

@@ -1,23 +1,23 @@
 #!/bin/bash
 #PBS -N borgms_run
-#PBS -l nodes=24:ppn=16
-#PBS -l walltime=33:00:00
+#PBS -l nodes=2:ppn=16
+#PBS -l walltime=2:00:00
 #PBS -o ./output/borgms_run_hybrid.out
 #PBS -e ./error/borgms_run_hybrid.err
 # #PBS -m bea
 # #PBS -M [email protected]
 module load openmpi-1.10.7-gnu-x86_64
-export OMP_NUM_THREADS=16
-N_REALIZATIONS=1000
-DATA_DIR="/scratch/bct52/"
+export OMP_NUM_THREADS=4
+N_REALIZATIONS=64
+DATA_DIR=${PBS_O_WORKDIR}/ #"/scratch/bct52/"
 
 set -x
 cd "$PBS_O_WORKDIR"
 
 # Construct a copy of the hostfile with only 16 entries per node.
 # MPI can use this to run 16 tasks on each node.
-export TASKS_PER_NODE=1
+export TASKS_PER_NODE=4
 uniq "$PBS_NODEFILE"|awk -v TASKS_PER_NODE="$TASKS_PER_NODE" '{for(i=0;i<TASKS_PER_NODE;i+=1) print}' > nodefile
 
 cat nodefile
-mpiexec --hostfile nodefile -n 24 -x OMP_NUM_THREADS ./triangleSimulation -T ${OMP_NUM_THREADS} -t 2344 -r ${N_REALIZATIONS} -d ${DATA_DIR} -C -1 -O rof_tables/ -b true -n 50000 -o 1000 -e 0
+mpiexec --hostfile nodefile -n 8 -x OMP_NUM_THREADS ./triangleSimulation -T ${OMP_NUM_THREADS} -t 2344 -r ${N_REALIZATIONS} -d ${DATA_DIR} -C -1 -O rof_tables_cac/ -b true -n 500 -o 100 -e 0

error/borgms_run_hybrid.err

@@ -0,0 +1,8 @@
+ModuleCmd_Load.c(200):ERROR:105: Unable to locate a modulefile for 'vim-8.0'
++ cd /home/fs02/pmr82_0001/bct52/wp_test/WaterPaths
++ export TASKS_PER_NODE=4
++ TASKS_PER_NODE=4
++ uniq /var/spool/torque/aux//485311.thecube.cac.cornell.edu
++ awk -v TASKS_PER_NODE=4 '{for(i=0;i<TASKS_PER_NODE;i+=1) print}'
++ cat nodefile
++ mpiexec --hostfile nodefile -n 8 -x OMP_NUM_THREADS ./triangleSimulation -T 4 -t 2344 -r 64 -d /home/fs02/pmr82_0001/bct52/wp_test/WaterPaths/ -C -1 -O rof_tables_cac/ -b true -n 500 -o 100 -e 0

nix/borg.nix

@@ -43,8 +43,8 @@ stdenv.mkDerivation rec {
     mkdir -p $out/lib
     mv *.exe $out/bin/
     mv *.a *.so $out/lib/
-    mv *.h $out/include/
-    # cp -R * $out # for debugging
+    cp *.h $out/include/
+    cp -R * $out/ # for debugging
   '';
 
 }

nix/remora.nix

@@ -0,0 +1,73 @@
+with import <nixpkgs> {};
+let
+  mpiP = stdenv.mkDerivation {
+    name = "mpiP";
+
+    src = fetchurl {
+      url = https://sourceforge.net/projects/mpip/files/mpiP/mpiP-3.4.1/mpiP-3.4.1.tar.gz;
+      sha256 = "1w4n693im81qgbq0m8f0cv8nd8l2h8srriczby8nl7i1fdyz72v8";
+    };
+    phases = "installPhase";
+
+    buildInputs = with stdenv; [ binutils gcc gfortran openmpi ];
+
+    installPhase = ''
+      mkdir -p $out/
+      tar -C $out -xzf $src
+
+      cd $out/mpiP-3.4.1/
+      ./configure
+      make
+      make install
+    '';
+  };
+  remora = stdenv.mkDerivation {
+    name = "remora";
+
+    src = fetchurl {
+      url = https://github.com/TACC/remora/archive/v1.8.3.tar.gz;
+      sha256 = "08n72i6r8y14zr5wlcllpccv4q9xz6lf0j2d5k7kmyz55qid8kb8";
+    };
+    phases = "installPhase";
+
+    buildInputs = with stdenv; [ binutils gcc gfortran openmpi ];
+
+    installPhase = ''
+      source $stdenv/setup
+
+      mkdir -p $out/
+      tar -C $out -xzf $src
+
+      export REMORA_INSTALL_PREFIX=$out
+      cd $out/remora-1.8.3
+      $out/remora-1.8.3/install.sh
+    '';
+  };
+in
+stdenv.mkDerivation {
+  name = "ProfilingEnv";
+  buildInputs = [
+    nix
+    bash
+    vim
+
+    # MPI-related packages
+    binutils
+    gcc
+    gfortran
+    openmpi
+    openssh
+
+    #
+    # Project code dependencies
+    #
+    mpiP
+    remora
+  ];
+  src = null;
+  shellHook = ''
+    export PATH=$PATH:${remora}/bin
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${remora}/lib
+    export REMORA_BIN=${remora}/bin
+  '';
+}

nix/shell.nix

@@ -35,6 +35,7 @@ stdenv.mkDerivation {
   shellHook = ''
     mkdir -p ./Borg
     ln -sfn ${Borg}/include/borgms.h ./Borg/borgms.h
+    ln -sfn ${Borg}/* ./Borg/
   '';
 }