Added hybrid MPI+OpenMP test in CI

tests/jenkins/Jenkinsfile

@@ -132,6 +132,13 @@ pipeline {
                         }
                     }
                 }
+                stage('nrntraub'){
+                    steps{
+                        dir('nrntraub'){
+                            git branch: 'icei', url: 'https://github.com/pramodk/nrntraub.git'
+                        }
+                    }
+                }
             }
         }
 
@@ -152,6 +159,11 @@ pipeline {
                         sh 'sh tests/jenkins/nrnivmodl.sh tqperf'
                     }
                 }
+                stage('nrnivmodl nrntraub'){
+                    steps{
+                        sh 'sh tests/jenkins/nrnivmodl.sh nrntraub'
+                    }
+                }
                 stage('nrnivmodl-core testcorenrn AoS'){
                     steps{
                         sh 'sh tests/jenkins/nrnivmodl-core.sh testcorenrn AoS'
@@ -182,6 +194,16 @@ pipeline {
                         sh 'sh tests/jenkins/nrnivmodl-core.sh tqperf SoA'
                     }
                 }
+                stage('nrnivmodl-core nrntraub AoS'){
+                    steps{
+                        sh 'sh tests/jenkins/nrnivmodl-core.sh nrntraub AoS'
+                    }
+                }
+                stage('nrnivmodl-core nrntraub SoA'){
+                    steps{
+                        sh 'sh tests/jenkins/nrnivmodl-core.sh nrntraub SoA'
+                    }
+                }
             }
         }
 
@@ -392,6 +414,25 @@ pipeline {
                         }
                     }
                 }
+                stage('nrntraub'){
+                    stages{
+                        stage('neuron'){
+                            steps{
+                                sh 'sh tests/jenkins/run_neuron.sh nrntraub nrntraub 4 9'
+                            }
+                        }
+                        stage('corenrn AoS'){
+                            steps{
+                                sh 'sh tests/jenkins/run_nrntraub.sh AoS 4 9'
+                            }
+                        }
+                        stage('corenrn SoA'){
+                            steps{
+                                sh 'sh tests/jenkins/run_nrntraub.sh SoA 4 9'
+                            }
+                        }
+                    }
+                }
             }
         }
     }

tests/jenkins/_test_utils.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+bb5_run() (
+    set +x -e
+    # default partition is interactive. during night use production
+    hour=`date +%H`
+    weekday=`date +%u`
+    if [ "$hour" -ge "19" ] || [ "$hour" -lt "8" ] || [ $weekday -gt 5 ]; then export SALLOC_PARTITION="prod"; fi
+
+    N=${N:-1}
+    if [ -n "$n" ]; then
+        SALLOC_OPTS="$SALLOC_OPTS --ntasks-per-node=$n"
+    else
+        SALLOC_OPTS="$SALLOC_OPTS --ntasks-per-node=36"
+    fi
+
+    cmd_base="time salloc -N$N $SALLOC_OPTS -Aproj16 --hint=compute_bound -Ccpu|nvme --time 1:00:00 srun dplace "
+    echo "$cmd_base $@"
+    $cmd_base "$@"
+)

tests/jenkins/run_neuron.sh

@@ -2,11 +2,13 @@
 
 set -e
 source ${JENKINS_DIR:-.}/_env_setup.sh
+source ${JENKINS_DIR:-.}/_test_utils.sh
 
 set -x
 TEST_DIR="$1"
 TEST="$2"
 MPI_RANKS="$3"
+THREADS="$4"
 
 cd $WORKSPACE/${TEST_DIR}
 
@@ -24,6 +26,11 @@ elif [ "${TEST_DIR}" = "tqperf" ]; then
     mkdir ${TEST}
     mpirun -n ${MPI_RANKS} ./x86_64/special -c tstop=50 run.hoc -mpi
     cat spk000.dat | sort -k 1n,1n -k 2n,2n > ${TEST}/out_nrn_${TEST}.spk
+elif [ "${TEST_DIR}" = "nrntraub" ]; then
+    # Allocate 1 node
+    N=1
+    n=${MPI_RANKS} bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=0 -c nthread=${THREADS} init.hoc
+    sort -n -k'1,1' -k2 < out${MPI_RANKS}.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_nrn.sorted
 else
     echo "Not a valid TEST"
     exit 1

tests/jenkins/run_nrntraub.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/bash
+
+set -e
+source ${JENKINS_DIR:-.}/_env_setup.sh
+source ${JENKINS_DIR:-.}/_test_utils.sh
+
+set -x
+
+CORENRN_TYPE="$1"
+MPI_RANKS="$2"
+THREADS="$3"
+
+cd $WORKSPACE/nrntraub
+
+# Allocate 1 node
+N=1
+
+export CORENEURONLIB=$(pwd)/${CORENRN_TYPE}/libcorenrnmech.so
+
+n=${MPI_RANKS} bb5_run ./x86_64/special -mpi -c mytstop=100 -c use_coreneuron=1 -c nthread=${THREADS} init.hoc
+
+sort -n -k'1,1' -k2 < out.dat | awk 'NR==1 { print; next } { printf "%.3f\t%d\n", $1, $2 }' > out_cn_${CORENRN_TYPE}.sorted
+
+sdiff -s out_cn_${CORENRN_TYPE}.sorted out_nrn.sorted