ld_neo4j_in_docker

#!/bin/bash
#
# Script for loading neo4j database 
#
# *** MAKE SURE YOU READ THIS SCRIPT BEFORE YOU RUN IT
# *** script removes directories (clean up)
#

# this script requires environment variables set in shell
# (for security reasons, these are not set in the script) 
#
# NEO4J_AUTH
# NEO4J_USERNAME
# NEO4J_PASSWORD
#
# configure build settings HERE
#
####################################################
CONTAINER="dclneodb"
DOCKER_IMAGE="neo4j:4.2.0"
EPHEMERALCONTAINER="ephemeral-${CONTAINER}"

NEO4J_DATA_PATH="."
NEO4J_GRAPH_DIR="$HOME/neo4j/${CONTAINER}"
NEO4J_DATA_SRC="${NEO4J_DATA_PATH}/"

# https://stackoverflow.com/questions/29278743/how-to-check-if-multiple-variables-are-defined-or-not-in-bash
check_vars()
{
    var_names=("$@")
    for var_name in "${var_names[@]}"; do
        [ -z "${!var_name}" ] && echo "$var_name is unset." && var_unset=true
    done
    [ -n "$var_unset" ] && exit 1
    return 0
}

check_vars NEO4J_USERNAME NEO4J_PASSWORD NEO4J_AUTH NEO4J_HTTPS NEO4J_HTTP NEO4J_BOLT

# neo4j memory settings
NEO4J_dbms_memory_heap_initial__size=5900m
NEO4J_dbms_memory_heap_max__size=5900m
NEO4J_dbms_memory_pagecache_size=3300m
NEO4J_dbms_memory_off__heap_max__size=4600m
NEO4J_dbms_jvm_additional="-XX:+ExitOnOutOfMemoryError"

NEO4J_PLUGINS='["apoc", "graph-data-science"]'

####################################################

echo "HOME: ${HOME}"
echo "Neo4j DB Config:"
echo "  NEO4J_HOST=${NEO4J_HOST}"
echo "  NEO4J_HTTPS=${NEO4J_HTTPS}"
echo "  NEO4J_HTTP=${NEO4J_HTTP}"
echo "  NEO4J_BOLT=${NEO4J_BOLT}"
echo "  NEO4J_dbms_memory_heap_initial__size=${NEO4J_dbms_memory_heap_initial__size}"
echo "  NEO4J_dbms_memory_heap_max__size=${NEO4J_dbms_memory_heap_max__size}"
echo "  NEO4J_dbms_memory_pagecache_size=${NEO4J_dbms_memory_pagecache_size}"
echo "  NEO4J_dbms_memory_off__heap_max__size=${NEO4J_dbms_memory_off__heap_max__size}"
echo "  NEO4J_dbms_jvm_additional=${NEO4J_dbms_jvm_additional}"
echo "Neo4j storage:"
echo "  NEO4J_DATA_SRC=${NEO4J_DATA_SRC}"
echo "  NEO4J_GRAPH_DIR=${NEO4J_GRAPH_DIR}"

read -p "Are the settings and directories correct? (type yes, to continue) "
if [ "$REPLY" != "yes" ]; then
   echo "Ok, I'm glad we checked the settings first!"
   exit
fi

# bash error trap
abort()
{
  local rv=$?
  shift
  [[ $rv ]] &&
   ((rv != 0)) && {
       echo "ERROR $@ ${rv} on line $(caller)"
       exit "$rv"
   }
}

#---------------------------------------------------------------
#
# to ensure there is no conflict, we proactively
# shutdown this docker container and remove it
#
#---------------------------------------------------------------

docker stop "${CONTAINER}"
docker rm "${CONTAINER}"

echo "rm -Rf ${NEO4J_GRAPH_DIR}/"
read -p "removing directory as shown above, please double check, is this OK?! (type yes, to continue) "
if [ "$REPLY" != "yes" ]; then
   echo "Ok, I'm glad asked first!"
   exit
fi

# cleanup old data? 
rm -Rf "${NEO4J_GRAPH_DIR}/"

echo "create directory structure required"
mkdir -p -m 700  "${NEO4J_GRAPH_DIR}/data"
mkdir -p -m 700  "${NEO4J_GRAPH_DIR}/import"

echo "copy data files to import directory"
cp -pr "${NEO4J_DATA_SRC}/e" "${NEO4J_GRAPH_DIR}/import/"
cp -pr "${NEO4J_DATA_SRC}/v" "${NEO4J_GRAPH_DIR}/import/"

echo "copy in post-load cypher scripts"
cp "${NEO4J_DATA_PATH}/"*.cypher "${NEO4J_GRAPH_DIR}/import/"

# run a bare bones ephemeral neo4j in a docker container
# this is required to use neo4j-admin on our new database
# because neo4j-admin import can not be used on a running neo4j
#

echo "stop/rm ${EPHEMERALCONTAINER} (if running)"
docker stop ${EPHEMERALCONTAINER}
docker rm ${EPHEMERALCONTAINER}

echo "start ${EPHEMERALCONTAINER} neo4j"
docker run \
  --user "$(id -u):$(id -g)" \
  --name "${EPHEMERALCONTAINER}" \
  --detach \
  --env NEO4J_AUTH="${NEO4J_AUTH}" \
  --env NEO4J_dbms_connector_https_listen__address=0.0.0.0:${NEO4J_HTTPS} \
  --env NEO4J_dbms_connector_http_listen__address=0.0.0.0:${NEO4J_HTTP} \
  --env NEO4J_dbms_connector_bolt_listen__address=0.0.0.0:${NEO4J_BOLT} \
  --env NEO4J_dbms_default__listen__address=0.0.0.0 \
  --env NEO4J_dbms_default__advertised__address=${NEO4J_HOST} \
  --env NEO4J_dbms_connector_https_advertised__address=${NEO4J_HOST}:${NEO4J_HTTPS} \
  --env NEO4J_dbms_connector_http_advertised__address=${NEO4J_HOST}:${NEO4J_HTTP} \
  --env NEO4J_dbms_connector_bolt_advertised__address=${NEO4J_HOST}:${NEO4J_BOLT} \
  --volume ${NEO4J_GRAPH_DIR}/data:/data \
  --volume ${NEO4J_GRAPH_DIR}/import:/import \
  --env NEO4J_dbms_memory_heap_initial__size=${NEO4J_dbms_memory_heap_initial__size} \
  --env NEO4J_dbms_memory_heap_max__size=${NEO4J_dbms_memory_heap_max__size} \
  --env NEO4J_dbms_memory_pagecache_size=${NEO4J_dbms_memory_pagecache_size} \
   ${DOCKER_IMAGE} 

abort $? "create ephemeral"

# wait for ephemeral neo4j to go fully online before proceeding
# note: ephemeral is always on the default port
./wait4bolt_outside ${EPHEMERALCONTAINER} ${NEO4J_BOLT} 

abort $? "ephemeral online"

echo "Bulk load all tsv files"
docker exec -t \
   ${EPHEMERALCONTAINER} bin/neo4j-admin import \
  --database="${CONTAINER}" \
  --verbose \
  --skip-bad-relationships=true \
  --skip-duplicate-nodes=true \
  --ignore-empty-strings=true \
  --normalize-types=true \
  --trim-strings=false \
  --delimiter "\t" \
  --high-io=true \
  --report-file=importreport.txt \
  --nodes=Cell="/import/v/cell.properties.drugcentral_lincs.hdr.tsv,/import/v/cell.properties.drugcentral_lincs.tsv.gz" \
  --nodes=Gene="/import/v/gene.properties.drugcentral_lincs.hdr.tsv,/import/v/gene.properties.drugcentral_lincs.tsv.gz" \
  --nodes=Signature="/import/v/signature.properties.drugcentral_lincs.hdr.tsv,/import/v/signature.properties.drugcentral_lincs.tsv.gz" \
  --nodes=Drug="/import/v/drug.properties.drugcentral_lincs.hdr.tsv,/import/v/drug.properties.drugcentral_lincs.tsv.gz" \
  --nodes=Concept="/import/v/concept.properties.drugcentral.hdr.tsv,/import/v/concept.properties.drugcentral.tsv.gz" \
  --relationships=CELL_SIGNATURE="/import/e/cell.signature.cellsignature.drugcentral_lincs.hdr.tsv,/import/e/cell.signature.cellsignature.drugcentral_lincs.tsv.gz" \
  --relationships=DRUG_SIGNATURE="/import/e/drug.signature.drugsignature.drugcentral_lincs.hdr.tsv,/import/e/drug.signature.drugsignature.drugcentral_lincs.tsv.gz" \
  --relationships=SIGNATURE_GENE="/import/e/signature.gene.signaturegene.drugcentral_lincs.hdr.tsv,/import/e/signature.gene.signaturegene.drugcentral_lincs.tsv.gz" \
  --relationships=INDICATION="/import/e/drug.concept.indication.drugcentral.hdr.tsv,/import/e/drug.concept.indication.drugcentral.tsv.gz"

abort $? "load graph"

echo "grab the error log, before we drop ephemeral"
docker exec -t ${EPHEMERALCONTAINER} bash -c "cp /var/lib/neo4j/importreport.txt /import/"

abort $? "grab importreport.txt"

echo "dropping ${EPHEMERALCONTAINER} neo4j"
docker stop ${EPHEMERALCONTAINER}
docker rm ${EPHEMERALCONTAINER}

#
# now start up neo4j and use the database we just created 
#
echo "starting neo4j"
docker run \
  --user $(id -u):$(id -g) \
  --name "${CONTAINER}" \
  --detach \
  --restart unless-stopped \
  --publish ${NEO4J_HTTPS}:${NEO4J_HTTPS} \
  --publish ${NEO4J_HTTP}:${NEO4J_HTTP} \
  --publish ${NEO4J_BOLT}:${NEO4J_BOLT} \
  --env NEO4J_metrics_jmx_enabled=true \
  --env NEO4J_dbms_connector_https_listen__address=0.0.0.0:${NEO4J_HTTPS} \
  --env NEO4J_dbms_connector_http_listen__address=0.0.0.0:${NEO4J_HTTP} \
  --env NEO4J_dbms_connector_bolt_listen__address=0.0.0.0:${NEO4J_BOLT} \
  --env NEO4J_dbms_connector_https_advertised__address=${NEO4J_HOST}:${NEO4J_HTTPS} \
  --env NEO4J_dbms_connector_http_advertised__address=${NEO4J_HOST}:${NEO4J_HTTP} \
  --env NEO4J_dbms_connector_bolt_advertised__address=${NEO4J_HOST}:${NEO4J_BOLT} \
  --volume ${NEO4J_GRAPH_DIR}/data:/data \
  --volume ${NEO4J_GRAPH_DIR}/import:/import \
  --env NEO4JLABS_PLUGINS="${NEO4J_PLUGINS}" \
  --env NEO4J_dbms_default__database="${CONTAINER}" \
  --env NEO4J_apoc_import_file_enabled=true \
  --env NEO4J_dbms_memory_heap_initial__size=${NEO4J_dbms_memory_heap_initial__size} \
  --env NEO4J_dbms_memory_heap_max__size=${NEO4J_dbms_memory_heap_max__size} \
  --env NEO4J_dbms_memory_pagecache_size=${NEO4J_dbms_memory_pagecache_size} \
  --env NEO4J_dbms_memory_off__heap_max__size=${NEO4J_dbms_memory_off__heap_max__size} \
  --env NEO4J_dbms_jvm_additional=${NEO4J_dbms_jvm_additional} \
   "${DOCKER_IMAGE}" 

abort $? "start graph"

# wait for neo4j, on correct port! 
./wait4bolt_outside ${CONTAINER} ${NEO4J_BOLT} 

echo "CREATE Indexes"
docker exec -e NEO4J_USERNAME -e NEO4J_PASSWORD -t ${CONTAINER} bash -c "bin/cypher-shell -a bolt://localhost:${NEO4J_BOLT} --format verbose --debug </import/indexes.cypher"