Enable the liveness probe via a checking of helm client.

main.go

@@ -102,7 +102,7 @@ func main() {
 	//Start a monitor for symphony operator
 	//monitorErrCh := make(chan error)
 	go func() {
-		mux := monitor.NewProbesMux()
+		mux := monitor.NewProbesMux(helmClient)
 
 		// Register gRPC server to prometheus to initialized matrix
 		//goprom.Register(rootServer)

pkg/helm/client.go

@@ -71,13 +71,14 @@ func (helmClient *Client) Ping() {
 	klog.Infof("Ping tiller")
 }
 
-func (helmClient *Client) KeepLive() {
+func (helmClient *Client) KeepLive() error {
 	versionResponse, err := helmClient.Client.GetVersion()
 	if err != nil {
 		klog.Errorf("Get version has an error in a keeping live process, %s", err.Error())
-		return
+		return err
 	}
 	klog.Infof("Keep the helm client tunnel alive regularly by sending getVersion request, version : %s", versionResponse.Version.SemVer)
+	return nil
 }
 
 //

pkg/monitor/probes.go

@@ -2,6 +2,7 @@ package monitor
 
 import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/yangyongzhi/sym-operator/pkg/helm"
 	"net/http"
 )
 
@@ -10,22 +11,40 @@ const MonitorAddr = ":44100"
 const (
 	isReadyMessage = "I am health."
 	isAliveMessage = "I am alive."
+	notLiveMessage = "The container is not live now."
 )
 
-func readinessProbe(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte(isReadyMessage))
+type LivenessHandler struct {
+	helmClient *helm.Client
+}
+
+func NewLivenessHandler(helmClient *helm.Client) *LivenessHandler {
+	livenessHandler := &LivenessHandler{
+		helmClient: helmClient,
+	}
+	return livenessHandler
 }
 
-func livenessProbe(w http.ResponseWriter, r *http.Request) {
+func (l *LivenessHandler) livenessProbe(w http.ResponseWriter, r *http.Request) {
+	err := l.helmClient.KeepLive()
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(notLiveMessage))
+	} else {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(isAliveMessage))
+	}
+}
+
+func readinessProbe(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
-	w.Write([]byte(isAliveMessage))
+	w.Write([]byte(isReadyMessage))
 }
 
-func NewProbesMux() *http.ServeMux {
+func NewProbesMux(helmClient *helm.Client) *http.ServeMux {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/readiness", readinessProbe)
-	mux.HandleFunc("/liveness", livenessProbe)
+	mux.HandleFunc("/liveness", NewLivenessHandler(helmClient).livenessProbe)
 	return mux
 }