Nautilus data pipeline fixes

Author: himanchali

.dockerignore

@@ -12,6 +12,9 @@
 !epf/Dockerfile
 !epf/src
 
+!images/experiment/Dockerfile
+!images/experiment/src
+
 !nautilus
 /nautilus/docker-compose.yml
 

Dockerfile

@@ -225,6 +225,7 @@ RUN sed -i "/gzip on;/d" /etc/nginx/conf.d/default.conf
 # Wrapper to handle additional script to run after default gitlab image's /assets/wrapper
 ADD nautilus/assets/ /assets
 ADD epf/ /epf
+ADD images/ /images
 
 # Export derived env variables from the above defined ENV variables.These should not be directly overwritten.
 RUN cat /assets/dynamic-env > /etc/bash.bashrc

images/experiment/Dockerfile

@@ -4,9 +4,9 @@ MAINTAINER MLReef
 ########## MLREEF ##########
 
 RUN apk add --no-cache \
-	git \
+    git  \
     curl \
-	wget \
+    wget \
     jq 
 
 ##### ADD files to the image

nautilus/assets/dynamic-env

@@ -25,5 +25,5 @@ export MLREEF_PG_LOG="/var/log/${MLREEF_PG_CLUSTER}-postgresql"
 
 ### Backend dynamic variables created from ENV mentioned in Dockerfile
 export EPF_BACKEND_URL=http://localhost:$MLREEF_BACKEND_PORT
-export EPF_GITLAB_URL=http://localhost
+export EPF_GITLAB_URL=http://localhost:10080
 

nautilus/assets/mlreef-wrapper

@@ -47,6 +47,17 @@ getGitlabRunnerRegistrationToken() {
   gitlab-rails runner -e production "puts Gitlab::CurrentSettings.current_application_settings.runners_registration_token" | tr -d '\r'
 }
 
+# Set max_artifacts_size using api
+setMaxArtifactSizeForGitlab() {
+  curl --request PUT --header "PRIVATE-TOKEN: $GITLAB_ADMIN_TOKEN" "${INSTANCE_HOST}:${GITLAB_PORT}"/"api/v4/application/settings?max_artifacts_size=5120"
+}
+
+# Set container_registry_token_expire_delay api
+setContainerRegistryTokenExpireDelayForGitlab() {
+  curl --request PUT --header "PRIVATE-TOKEN: $GITLAB_ADMIN_TOKEN" "${INSTANCE_HOST}:${GITLAB_PORT}"/"api/v4/application/settings?container_registry_token_expire_delay=30"
+}
+
+
 #########################
 # Call the Gitlab wrapper
 # Gitlab will self-daemonize and run in the background hence no '&' is needed
@@ -69,6 +80,12 @@ log "Expecting code 302; received: $(checkGitlabPort /runners)"
 log "Refreshing root user api-token"
 recreateGitlabAdminToken
 
+#########################
+# Gitlab setting changes
+log "Changing gitlab settings"
+setMaxArtifactSizeForGitlab
+setContainerRegistryTokenExpireDelayForGitlab
+
 log "Getting Gitlab runners registration token from Gitlab."
 RUNNER_REGISTRATION_TOKEN=$(getGitlabRunnerRegistrationToken)
 
@@ -136,11 +153,16 @@ echo "Project Id: $projectId"
 export MLREEF_DOCKER_REGISTRY_USER=root
 export MLREEF_DOCKER_REGISTRY_IMAGE="${MLREEF_DOCKER_REGISTRY}/${GITLAB_ROOTGROUP}/${GITLAB_PROJECT}"
 export MLREEF_DOCKER_REGISTRY_EPF_IMAGE=$MLREEF_DOCKER_REGISTRY_IMAGE/epf:latest
+export MLREEF_DOCKER_REGISTRY_EXPERIMENT_IMAGE=$MLREEF_DOCKER_REGISTRY_IMAGE/experiment:latest
 
 if [ -n "${MLREEF_DOCKER_REGISTRY_EXTERNAL_URL+x}" ] && [ -n "${MLREEF_DOCKER_REGISTRY_USER+x}" ] && [ -n "${GITLAB_ADMIN_TOKEN+x}" ]; then
           echo "$GITLAB_ADMIN_TOKEN" | docker login "$MLREEF_DOCKER_REGISTRY_EXTERNAL_URL" --username="$MLREEF_DOCKER_REGISTRY_USER" --password-stdin
 fi
 
+docker build --tag "$MLREEF_DOCKER_REGISTRY_EXPERIMENT_IMAGE" -f ../images/experiment/Dockerfile ../images/experiment
+docker push "$MLREEF_DOCKER_REGISTRY_EXPERIMENT_IMAGE"
+log "$MLREEF_DOCKER_REGISTRY_EXPERIMENT_IMAGE has been successfully uploaded "
+
 docker build --tag "$MLREEF_DOCKER_REGISTRY_EPF_IMAGE" -f ../epf/Dockerfile ../epf
 docker push "$MLREEF_DOCKER_REGISTRY_EPF_IMAGE"
 log "$MLREEF_DOCKER_REGISTRY_EPF_IMAGE has been successfully uploaded "