Fix the readiness probe using a sidecar container

charts/ibm-mq/templates/role-binding.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.readinessProbe.useLabelPatcher }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "ibm-mq.fullname" ( . ) }}-label-patcher-binding
+  labels:
+    {{- include "ibm-mq.labels" . | nindent 4 }}
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: {{ include "ibm-mq.fullname" ( . ) }}-label-patcher
+subjects:
+- kind: ServiceAccount
+  name: {{ include "ibm-mq.fullname" ( . ) }}
+  namespace: {{ .Release.Namespace }}
+{{ end -}}

charts/ibm-mq/templates/role.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.readinessProbe.useLabelPatcher }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "ibm-mq.fullname" ( . ) }}-label-patcher
+  labels:
+    {{- include "ibm-mq.labels" . | nindent 4 }}
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  resourceNames: [
+    {{ include "ibm-mq.pod0.name" . }},
+    {{- if .Values.queueManager.nativeha.enable }}
+    {{ include "ibm-mq.pod1.name" . }},
+    {{ include "ibm-mq.pod2.name" . }}
+    {{- end }}
+    ]
+  verbs: ["get", "list","patch"]
+{{ end -}}

charts/ibm-mq/templates/service-loadbalancer.yaml

@@ -43,4 +43,7 @@ spec:
   {{- end }}
   selector:
 {{- include "ibm-mq.selectorLabels" . | nindent 4 }}
+    {{- if .Values.readinessProbe.useLabelPatcher }}
+    role: master
+    {{- end }}
 {{- end }}

charts/ibm-mq/templates/service-qm.yaml

@@ -25,4 +25,7 @@ spec:
     name: qmgr
   selector:
 {{- include "ibm-mq.selectorLabels" . | nindent 4 }}
+    {{- if .Values.readinessProbe.useLabelPatcher }}
+    role: master
+    {{- end }}
 {{- end }}

charts/ibm-mq/templates/service-web.yaml

@@ -25,4 +25,7 @@ spec:
     name: console-https
   selector:
 {{- include "ibm-mq.selectorLabels" . | nindent 4 }}
+    {{- if .Values.readinessProbe.useLabelPatcher }}
+    role: master
+    {{- end }}
 {{- end }}

charts/ibm-mq/templates/service.yaml

@@ -26,3 +26,6 @@ spec:
     name: qmgr
   selector:
 {{- include "ibm-mq.selectorLabels" . | nindent 4 }}
+    {{- if .Values.readinessProbe.useLabelPatcher }}
+    role: master
+    {{- end }}

charts/ibm-mq/templates/stateful-set.yaml

@@ -135,6 +135,8 @@ spec:
       volumes:
       {{- else if .Values.web.manualConfig.configMap.name }}
       volumes:
+      {{- else if .Values.readinessProbe.useLabelPatcher }}
+      volumes:
       {{- else if .Values.web.manualConfig.secret.name }}
       volumes:
       {{- end}}
@@ -272,8 +274,49 @@ spec:
           defaultMode: 420
           secretName: {{ .Values.credentials.secret }}
       {{- end }}
+      {{- if .Values.readinessProbe.useLabelPatcher }}
+      - name: state
+        emptyDir: {}
+      {{- end }}
       terminationGracePeriodSeconds: {{.Values.queueManager.terminationGracePeriodSeconds}}
       containers:
+        {{- if .Values.readinessProbe.useLabelPatcher }}
+        - name: state-checker
+          command:
+            - sh
+            - '-c'
+            - >
+              sleep 20;
+              while true; do
+                sleep $(($RANDOM % 5 + 5));
+                KUBE_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+                NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
+                echo "Checking state of the queue manager"
+                STATE=$(cat /etc/mqm/state/test)
+                if [ $STATE -eq 0 ]; then
+                echo "Queue manager is active"
+                echo "Patching the label"
+                curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" \
+                    --request PATCH \
+                    --header "Content-Type: application/json-patch+json" \
+                    --data '[ { "op": "replace", "path": "/metadata/labels/role", "value": "master" } ]' \
+                    https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/$NAMESPACE/pods/$HOSTNAME > /dev/null
+                else
+                echo "Queue manager is not active"
+                echo "Patching the label"
+                curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" \
+                    --header "Content-Type: application/json-patch+json" \
+                    --request PATCH \
+                    --data '[ { "op": "replace", "path": "/metadata/labels/role", "value": "standby" } ]' \
+                    https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/$NAMESPACE/pods/$HOSTNAME > /dev/null
+                fi
+              done
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - mountPath: /etc/mqm/state
+              name: state
+        {{- end }}
         - name: qmgr
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
@@ -370,6 +413,8 @@ spec:
           volumeMounts:
           {{- else if .Values.web.manualConfig.secret.name }}
           volumeMounts:
+          {{- else if .Values.readinessProbe.useLabelPatcher }}
+          volumeMounts:
           {{- end}}
           {{- if .Values.queueManager.nativeha.tls }}
           {{- if .Values.queueManager.nativeha.tls.secretName }}
@@ -465,6 +510,10 @@ spec:
           - name: mq-credentials
             mountPath: "/var/run/secrets"
           {{- end }}
+          {{- if .Values.readinessProbe.useLabelPatcher }}
+          - name: state
+            mountPath: /etc/mqm/state
+          {{- end }}
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: {{ .Values.security.readOnlyRootFilesystem }}
@@ -492,8 +541,17 @@ spec:
           # Set readiness probe to determine if the MQ listener is running
           readinessProbe:
             exec:
+              {{- if .Values.readinessProbe.useLabelPatcher }}
+              command:
+              - sh
+              - '-c'
+              - >
+                chkmqready;
+                echo $? > /etc/mqm/state/test
+              {{- else }}
               command:
               - chkmqready
+              {{- end }}
             {{- if or .Values.queueManager.nativeha.enable .Values.queueManager.multiinstance.enable }}
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds | default 0 }}
             {{- else }}

charts/ibm-mq/values.yaml

@@ -139,6 +139,7 @@ livenessProbe:
 
 # readinessProbe section specifies setting for the MQ readiness probe, which checks when the MQ listener is running
 readinessProbe:
+  useLabelPatcher: false
   initialDelaySeconds:
   periodSeconds: 5
   timeoutSeconds: 3