Merge pull request #6 from sarjeet2013/master

Update KDF plugin to v1.1.0 release version
mapr · Dec 14, 2018 · 1ea5d30 · 1ea5d30
2 parents 64c15b7 + 60c3dc0
commit 1ea5d30
Show file tree

Hide file tree

Showing 20 changed files with 90 additions and 36 deletions.
diff --git a/build/plugincentos/Dockerfile b/build/plugincentos/Dockerfile
@@ -27,18 +27,20 @@ RUN rpm --import http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7; \
 WORKDIR /tmp
 
 # Main Docker Build
-RUN wget wget http://package.mapr.com/tools/KubernetesDataFabric/v1.0.2/redhat/mapr-client-6.0.1.20180710151556.GA-20180710151556.x86_64.rpm; \
-    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.0.1/redhat/mapr-librdkafka-0.11.3.201803231414-1.noarch.rpm; \
-    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.0.2/redhat/mapr-posix-client-basic-6.0.1.20180710151556.GA-1.x86_64.rpm; \
-    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.0.2/redhat/mapr-posix-client-platinum-6.0.1.20180710151556.GA-1.x86_64.rpm; \
+RUN wget http://package.mapr.com/tools/KubernetesDataFabric/v1.1.0/redhat/mapr-client-6.1.0.20180926230239.GA-1.x86_64.rpm; \
+    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.1.0/redhat/mapr-librdkafka-0.11.3.201803231414-1.noarch.rpm; \
+    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.1.0/redhat/mapr-posix-client-basic-6.1.0.20180926230239.GA-1.x86_64.rpm; \
+    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.1.0/redhat/mapr-posix-client-platinum-6.1.0.20180926230239.GA-1.x86_64.rpm; \
     rpm -Uvh mapr-librdkafka-0.11.3.201803231414-1.noarch.rpm; \
-    rpm -Uvh mapr-client-6.0.1.20180710151556.GA-20180710151556.x86_64.rpm; \
-    rpm -Uvh mapr-posix-client-basic-6.0.1.20180710151556.GA-1.x86_64.rpm; \
-    rpm -Uvh mapr-posix-client-platinum-6.0.1.20180710151556.GA-1.x86_64.rpm; \
+    rpm -Uvh mapr-client-6.1.0.20180926230239.GA-1.x86_64.rpm; \
+    rpm -Uvh mapr-posix-client-basic-6.1.0.20180926230239.GA-1.x86_64.rpm; \
+    rpm -Uvh mapr-posix-client-platinum-6.1.0.20180926230239.GA-1.x86_64.rpm; \
     mkdir -p /tmp/lib; \
+    mkdir -p /tmp/bin; \
     cp -r /opt/mapr/lib/libfuse.* /tmp/lib; \
     cp -r /opt/mapr/lib/libMapRClient_c.* /tmp/lib; \
     cp -r /opt/mapr/lib/libMapRClient.* /tmp/lib; \
+    cp /opt/mapr/server/mruuidgen /tmp/bin; \
     cp /usr/lib/jvm/jre/lib/amd64/server/libjvm.so /tmp/lib/libjvm.so; \
     rm -rf /opt/mapr/contrib; \
     rm -rf /opt/mapr/conf; \
@@ -56,6 +58,7 @@ RUN wget wget http://package.mapr.com/tools/KubernetesDataFabric/v1.0.2/redhat/m
     rm -rf /opt/mapr/bin/m*; \
     mkdir -p /opt/mapr/lib; \
     cp -r /tmp/lib/* /opt/mapr/lib; \
+    cp /tmp/bin/mruuidgen /opt/mapr/bin/; \
     rm -rf /tmp
 
 # Use Small Image

diff --git a/build/plugincentos/copy2host b/build/plugincentos/copy2host
@@ -75,6 +75,7 @@ cp -r $mapr_bin/fusermount $host_bin
 cp -r $mapr_bin/stubfuse $host_bin
 cp -r $mapr_bin/posix-client-platinum $host_bin
 cp -r $mapr_bin/posix-client-basic $host_bin
+cp -r $mapr_bin/mruuidgen $host_bin
 log "INFO  Copying support script: $host_bin/start-fuse"
 cp $docker_mapr/plugin/start-fuse $host_bin/start-fuse
 chmod 755 $host_bin/*

diff --git a/build/plugincentos/copy2mapr b/build/plugincentos/copy2mapr
@@ -100,6 +100,7 @@ if [ $versions_match -eq $no ]; then
     cp -r $mapr_bin/stubfuse $host_bin
     cp -r $mapr_bin/posix-client-platinum $host_bin
     cp -r $mapr_bin/posix-client-basic $host_bin
+    cp -r $mapr_bin/mruuidgen $host_bin
 fi
 log "INFO  Copying support scripts from: $docker_mapr/plugin/start-fuse to: $host_bin/start-fuse"
 cp $mapr_bin/start-fuse $host_bin/start-fuse

diff --git a/build/plugincentos/docker-build.sh b/build/plugincentos/docker-build.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-docker build --force-rm --pull -t maprtech/kdf-plugin:1.0.2_001_centos7 .
+docker build --force-rm --pull -t maprtech/kdf-plugin:1.1.0_001_centos7 .
diff --git a/build/plugincentos/maprfs/main.go b/build/plugincentos/maprfs/main.go
@@ -266,11 +266,11 @@ func startFuse(kpath string, options map[string]string) string {
 			os.Exit(1)
 		}
 		conf = cluster + " secure=true " + cldblist
-		Plugin.Printf("INFO  Creating cluster info: ", conf)
+		Plugin.Printf("INFO  Creating cluster info: %s", conf)
 	}
 	// Setup command to call FUSE
 	Plugin.Printf("INFO  Calling FUSE script: %s...", fuse_script)
-	cmd := exec.Command(fuse_script, fmount, spath, ffspath, kpath, sectype, ticketfile, conf, platinum, args)
+	cmd := exec.Command(fuse_script, fmount, spath, ffspath, kpath, sectype, ticketfile, conf, platinum, podid, args)
 	// Call FUSE
 	err = cmd.Run()
 	if err != nil {

diff --git a/build/plugincentos/start-fuse b/build/plugincentos/start-fuse
@@ -17,7 +17,13 @@ sec_type=$5
 ticket_location=$6
 config=$7
 platinum=$8
-flags=$9
+podid=$9
+flags=${10}
+
+# K8S-353: Run fuse in the process scope instead of kubelet.service scope.
+# Running it in kubelet.service scope will kill fuse process on kubelet restart.
+systemdrun_with_scope="/usr/bin/systemd-run --scope"
+
 conf_file="$support_path/mapr-clusters.conf"
 fuse_args="$mount_path -d --log_path $ffs_log_path --client_lib_path $support_path --conf_file $conf_file $flags"
 log() {
@@ -26,6 +32,17 @@ log() {
   message="$prefix$1"
   echo $message >> $fuse_log
 }
+
+# Use pod-id as the hostname for the posix-client instance
+HOSTNAME_FILE="$k8s_dir/hostname"
+log "INFO Setting Posix client hostname: ${podid}" >> $fuse_log
+echo $podid > ${HOSTNAME_FILE}
+ # Generate hostid for the posix client
+HOSTID_FILE="$k8s_dir/hostid"
+log "INFO Setting posix client hostid.." >> $fuse_log
+$mapr_bin/mruuidgen > ${HOSTID_FILE}
+chmod 444 ${HOSTID_FILE}
+
 log "INFO  export MAPR_HOME=$k8s_dir"
 export MAPR_HOME=$k8s_dir
 log "INFO  Setting ulimits... "
@@ -58,11 +75,13 @@ else
   cp "$mapr_lib_path/libMapRClient.so.1" "$support_path/libMapRClient.so.0"
 fi
 log "INFO  Mounting FUSE via POSIX client: $fuse_program $fuse_args"
-$fuse_program $fuse_args &
+$systemdrun_with_scope $fuse_program $fuse_args &
 pid=$!
 pidfile="$support_path/fusepid"
 log "INFO  Writing POSIX client pid: $pid to support directory: $pidfile..."
-echo $pid >> $pidfile
+# K8S-310: During restart, the duplicate mount can cause multiple fuse pid
+# to written to fusepid file.
+echo $pid > $pidfile
 pathfile="$support_path/kpath"
 log "INFO  Writing kpath: $kube_path to support directory: $pathfile..."
 echo $kube_path >> $pathfile
@@ -77,4 +96,4 @@ else
     log "ERROR  POSIX client process: $pid is unhealthy. Check ffs-log.* for more info"
     exit 1
 fi
-exit 0
+exit 0
diff --git a/build/pluginubuntu/Dockerfile b/build/pluginubuntu/Dockerfile
@@ -21,18 +21,20 @@ RUN apt-get update; \
     apt-get install -y apt-utils sudo lsb-release; \
     DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y wget perl syslinux openjdk-8-jdk
 # Install MapR packages
-RUN wget http://package.mapr.com/tools/KubernetesDataFabric/v1.0.2/ubuntu/mapr-client-6.0.1.20180710151556.GA-20180710151556.amd64.deb; \
-    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.0.1/ubuntu/mapr-librdkafka_0.11.3.201803231414_all.deb; \
-    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.0.2/ubuntu/mapr-posix-client-basic_6.0.1.20180710151556.GA-20180710151556_amd64.deb; \
-    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.0.2/ubuntu/mapr-posix-client-platinum_6.0.1.20180710151556.GA-20180710151556_amd64.deb; \
+RUN wget http://package.mapr.com/tools/KubernetesDataFabric/v1.1.0/ubuntu/mapr-client-6.1.0.20180926230239.GA-1.amd64.deb; \
+    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.1.0/ubuntu/mapr-librdkafka_0.11.3.201803231414_all.deb; \
+    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.1.0/ubuntu/mapr-posix-client-basic_6.1.0.20180926230239.GA-1_amd64.deb; \
+    wget http://package.mapr.com/tools/KubernetesDataFabric/v1.1.0/ubuntu/mapr-posix-client-platinum_6.1.0.20180926230239.GA-1_amd64.deb; \
     dpkg -i --force-overwrite /tmp/mapr-librdkafka_0.11.3.201803231414_all.deb; \
-    dpkg -i --force-overwrite /tmp/mapr-client-6.0.1.20180710151556.GA-20180710151556.amd64.deb; \
-    dpkg -i --force-overwrite /tmp/mapr-posix-client-basic_6.0.1.20180710151556.GA-20180710151556_amd64.deb; \
-    dpkg -i --force-overwrite /tmp/mapr-posix-client-platinum_6.0.1.20180710151556.GA-20180710151556_amd64.deb; \
+    dpkg -i --force-overwrite /tmp/mapr-client-6.1.0.20180926230239.GA-1.amd64.deb; \
+    dpkg -i --force-overwrite /tmp/mapr-posix-client-basic_6.1.0.20180926230239.GA-1_amd64.deb; \
+    dpkg -i --force-overwrite /tmp/mapr-posix-client-platinum_6.1.0.20180926230239.GA-1_amd64.deb; \
     mkdir -p /tmp/lib; \
+    mkdir -p /tmp/bin; \
     cp -r /opt/mapr/lib/libfuse.* /tmp/lib; \
     cp -r /opt/mapr/lib/libMapRClient_c.* /tmp/lib; \
     cp -r /opt/mapr/lib/libMapRClient.* /tmp/lib; \
+    cp /opt/mapr/server/mruuidgen /tmp/bin; \
     cp /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so /tmp/lib/libjvm.so; \
     rm -rf /opt/mapr/contrib; \
     rm -rf /opt/mapr/conf; \
@@ -50,6 +52,7 @@ RUN wget http://package.mapr.com/tools/KubernetesDataFabric/v1.0.2/ubuntu/mapr-c
     rm -rf /opt/mapr/bin/m*; \
     mkdir -p /opt/mapr/lib; \
     cp -r /tmp/lib/* /opt/mapr/lib; \
+    cp /tmp/bin/mruuidgen /opt/mapr/bin/; \
     rm -rf /tmp
 
 # Use Small Image. Main Docker Build

diff --git a/build/pluginubuntu/copy2host b/build/pluginubuntu/copy2host
@@ -75,6 +75,7 @@ cp -r $mapr_bin/fusermount $host_bin
 cp -r $mapr_bin/stubfuse $host_bin
 cp -r $mapr_bin/posix-client-platinum $host_bin
 cp -r $mapr_bin/posix-client-basic $host_bin
+cp -r $mapr_bin/mruuidgen $host_bin
 log "INFO  Copying support script: $host_bin/start-fuse"
 cp $docker_mapr/plugin/start-fuse $host_bin/start-fuse
 chmod 755 $host_bin/*

diff --git a/build/pluginubuntu/copy2mapr b/build/pluginubuntu/copy2mapr
@@ -100,6 +100,7 @@ if [ $versions_match -eq $no ]; then
     cp -r $mapr_bin/stubfuse $host_bin
     cp -r $mapr_bin/posix-client-platinum $host_bin
     cp -r $mapr_bin/posix-client-basic $host_bin
+    cp -r $mapr_bin/mruuidgen $host_bin
 fi
 log "INFO  Copying support scripts from: $docker_mapr/plugin/start-fuse to: $host_bin/start-fuse"
 cp $mapr_bin/start-fuse $host_bin/start-fuse

diff --git a/build/pluginubuntu/docker-build.sh b/build/pluginubuntu/docker-build.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-docker build --force-rm --pull -t maprtech/kdf-plugin:1.0.2_001_ubuntu .
+docker build --force-rm --pull -t maprtech/kdf-plugin:1.1.0_001_ubuntu .
diff --git a/build/pluginubuntu/maprfs/main.go b/build/pluginubuntu/maprfs/main.go
@@ -266,11 +266,11 @@ func startFuse(kpath string, options map[string]string) string {
 			os.Exit(1)
 		}
 		conf = cluster + " secure=true " + cldblist
-		Plugin.Printf("INFO  Creating cluster info: ", conf)
+		Plugin.Printf("INFO  Creating cluster info: %s", conf)
 	}
 	// Setup command to call FUSE
 	Plugin.Printf("INFO  Calling FUSE script: %s...", fuse_script)
-	cmd := exec.Command(fuse_script, fmount, spath, ffspath, kpath, sectype, ticketfile, conf, platinum, args)
+	cmd := exec.Command(fuse_script, fmount, spath, ffspath, kpath, sectype, ticketfile, conf, platinum, podid, args)
 	// Call FUSE
 	err = cmd.Run()
 	if err != nil {

diff --git a/build/pluginubuntu/start-fuse b/build/pluginubuntu/start-fuse
@@ -17,7 +17,13 @@ sec_type=$5
 ticket_location=$6
 config=$7
 platinum=$8
-flags=$9
+podid=$9
+flags=${10}
+
+# K8S-353: Run fuse in the process scope instead of kubelet.service scope.
+# Running it in kubelet.service scope will kill fuse process on kubelet restart.
+systemdrun_with_scope="/usr/bin/systemd-run --scope"
+
 conf_file="$support_path/mapr-clusters.conf"
 fuse_args="$mount_path -d --log_path $ffs_log_path --client_lib_path $support_path --conf_file $conf_file $flags"
 log() {
@@ -26,6 +32,17 @@ log() {
   message="$prefix$1"
   echo $message >> $fuse_log
 }
+
+# Use pod-id as the hostname for the posix-client instance
+HOSTNAME_FILE="$k8s_dir/hostname"
+log "INFO Setting Posix client hostname: ${podid}" >> $fuse_log
+echo $podid > ${HOSTNAME_FILE}
+ # Generate hostid for the posix client
+HOSTID_FILE="$k8s_dir/hostid"
+log "INFO Setting posix client hostid.." >> $fuse_log
+$mapr_bin/mruuidgen > ${HOSTID_FILE}
+chmod 444 ${HOSTID_FILE}
+
 log "INFO  export MAPR_HOME=$k8s_dir"
 export MAPR_HOME=$k8s_dir
 log "INFO  Setting ulimits... "
@@ -58,11 +75,13 @@ else
   cp "$mapr_lib_path/libMapRClient.so.1" "$support_path/libMapRClient.so.0"
 fi
 log "INFO  Mounting FUSE via POSIX client: $fuse_program $fuse_args"
-$fuse_program $fuse_args &
+$systemdrun_with_scope  $fuse_program $fuse_args &
 pid=$!
 pidfile="$support_path/fusepid"
 log "INFO  Writing POSIX client pid: $pid to support directory: $pidfile..."
-echo $pid >> $pidfile
+# K8S-310: During restart, the duplicate mount can cause multiple fuse pid
+# to written to fusepid file.
+echo $pid > $pidfile
 pathfile="$support_path/kpath"
 log "INFO  Writing kpath: $kube_path to support directory: $pathfile..."
 echo $kube_path >> $pathfile
@@ -77,4 +96,4 @@ else
     log "ERROR  POSIX client process: $pid is unhealthy. Check ffs-log.* for more info"
     exit 1
 fi
-exit 0
+exit 0
diff --git a/build/provisioner/docker-build.sh b/build/provisioner/docker-build.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
- docker build --force-rm --pull -t maprtech/kdf-provisioner:1.0.0.008_centos7 .
+ docker build --force-rm --pull -t maprtech/kdf-provisioner:1.1.0.001_centos7 .
diff --git a/build/provisioner/maprfs-provisioner/main.go b/build/provisioner/maprfs-provisioner/main.go
@@ -36,7 +36,13 @@ import (
 var (
 	master      = flag.String("master", "", "Master URL")
 	kubeconfig  = flag.String("kubeconfig", "", "Absolute path to the kubeconfig")
-	legalparams = [...]string{"cldbHosts", "maprSecretName", "maprSecretNamespace", "ticketSecretName", "ticketSecretNamespace", "namePrefix", "cluster", "mount", "mountPrefix", "securityType", "restServers", "platinum", "mountOptions", "readOnly", "reclaimPolicy", "advisoryquota", "ae", "aetype", "allowgrant", "allowinherit", "auditenabled", "cluster", "coalesce", "containerallocationfactor", "criticalrereplicationtimeoutsec", "dataauditops", "dbindexlagsecalarmthresh", "dbrepllagsecalarmthresh", "enforceminreplicationforio", "forceauditenable", "group", "inherit", "localvolumehost", "localvolumeport", "maxinodesalarmthreshold", "maxnssizembalarmthreshold", "minreplication", "mirrorschedule", "mirrorthrottle", "nsminreplication", "nsreplication", "readAce", "readonly", "replication", "replicationtype", "rereplicationtimeoutsec", "rootdirperms", "schedule", "skipinherit", "source", "tenantuser", "topology", "type", "user", "wiresecurityenabled", "writeaAce"}
+	legalparams = [...]string{"cldbHosts", "maprSecretName", "maprSecretNamespace", "ticketSecretName", "ticketSecretNamespace", "namePrefix", "cluster", "mount", "mountPrefix", "securityType", "restServers",
+		"platinum", "mountOptions", "readOnly", "reclaimPolicy", "advisoryquota", "ae", "aetype", "allowgrant", "allowinherit", "auditenabled", "autooffloadthresholdgb", "cluster", "coalesce",
+		"compactionoverheadthreshold", "compactionschedule","containerallocationfactor", "criticalrereplicationtimeoutsec", "dare", "dataauditops", "dbindexlagsecalarmthresh", "dbrepllagsecalarmthresh",
+		"ecenable", "ecscheme", "ectopology", "enforceminreplicationforio", "forceauditenable", "group", "inherit", "localvolumehost", "localvolumeport", "maxinodesalarmthreshold", "maxnssizembalarmthreshold",
+		"metricsenabled", "minreplication", "mirrorschedule", "mirrorthrottle", "namecontainerdatathreshold", "nsminreplication", "nsreplication", "readAce", "readonly", "recallexpirytime", "replication",
+		"replicationtype", "rereplicationtimeoutsec", "rootdirperms", "rootdiruser", "schedule", "skipinherit", "source", "tenantuser", "tierencryption", "tieringenable", "tieringrule", "tierkey", "tiername",
+		"topology", "type", "user", "wiresecurityenabled", "writeaAce"}
 	Plog        *log.Logger
 )
 

diff --git a/deploy/kdf-plugin-azure.yaml b/deploy/kdf-plugin-azure.yaml
@@ -21,7 +21,7 @@ spec:
       containers:
         - name: mapr-kdfplugin
           imagePullPolicy: Always
-          image: maprtech/kdf-plugin:1.0.2_001_ubuntu
+          image: maprtech/kdf-plugin:1.1.0_001_ubuntu
           command:
           - bash
           - -c

diff --git a/deploy/kdf-plugin-centos.yaml b/deploy/kdf-plugin-centos.yaml
@@ -21,7 +21,7 @@ spec:
       containers:
         - name: mapr-kdfplugin
           imagePullPolicy: Always
-          image: maprtech/kdf-plugin:1.0.2_001_centos7
+          image: maprtech/kdf-plugin:1.1.0_001_centos7
           command:
           - bash
           - -c

diff --git a/deploy/kdf-plugin-gke.yaml b/deploy/kdf-plugin-gke.yaml
@@ -21,7 +21,7 @@ spec:
       containers:
         - name: mapr-kdfplugin
           imagePullPolicy: Always
-          image: maprtech/kdf-plugin:1.0.2_001_ubuntu
+          image: maprtech/kdf-plugin:1.1.0_001_ubuntu
           command:
           - bash
           - -c

diff --git a/deploy/kdf-plugin-openshift.yaml b/deploy/kdf-plugin-openshift.yaml
@@ -21,7 +21,7 @@ spec:
       containers:
         - name: mapr-kdfplugin
           imagePullPolicy: Always
-          image: maprtech/kdf-plugin:1.0.2_001_centos7
+          image: maprtech/kdf-plugin:1.1.0_001_centos7
           command:
           - bash
           - -c

diff --git a/deploy/kdf-plugin-ubuntu.yaml b/deploy/kdf-plugin-ubuntu.yaml
@@ -21,7 +21,7 @@ spec:
       containers:
         - name: mapr-kdfplugin
           imagePullPolicy: Always
-          image: maprtech/kdf-plugin:1.0.2_001_ubuntu
+          image: maprtech/kdf-plugin:1.1.0_001_ubuntu
           command:
           - bash
           - -c

diff --git a/deploy/kdf-provisioner.yaml b/deploy/kdf-provisioner.yaml
@@ -16,7 +16,7 @@ spec:
       serviceAccountName: maprkdf
       containers:
       - name: mapr-kdfprovisioner
-        image: maprtech/kdf-provisioner:1.0.0.008_centos7
+        image: maprtech/kdf-provisioner:1.1.0.001_centos7
         imagePullPolicy: Always
         command:
         - /maprfs-provisioner