RANGER-5227: Add healthchecks for all containers

Author: kumaab

.github/workflows/ci.yml

@@ -165,11 +165,14 @@ jobs:
           -f docker-compose.ranger-kafka.yml \
           -f docker-compose.ranger-hive.yml \
           -f docker-compose.ranger-knox.yml \
-          -f docker-compose.ranger-ozone.yml up -d
+          -f docker-compose.ranger-ozone.yml up -d --wait
+
+      - name: Debug Logs in OM Container
+        run: |
+          docker logs -f ozone-om || true
 
       - name: Check status of containers and remove them
         run: |
-          sleep 60
           containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode);
           flag=true;
           for container in "${containers[@]}"; do

dev-support/ranger-docker/docker-compose.ranger-hadoop.yml

@@ -23,8 +23,8 @@ services:
       ranger:
         condition: service_started
     healthcheck:
-      test: "hdfs dfs -ls /hbase"
-      interval: 1m30s
+      test: [ "CMD-SHELL", "su hdfs -c \"/opt/hadoop/bin/hdfs dfsadmin -report | grep -q 'Live datanodes'\"" ]
+      interval: 30s
       timeout: 10s
       retries: 30
       start_period: 40s

dev-support/ranger-docker/docker-compose.ranger-hbase.yml

@@ -25,6 +25,12 @@ services:
         condition: service_healthy
       ranger-zk:
         condition: service_started
+    healthcheck:
+      test: [ "CMD", "wget", "-q", "--spider", "http://localhost:16030/rs-status" ]
+      interval: 30s
+      timeout: 10s
+      retries: 30
+      start_period: 40s
     environment:
       - HBASE_VERSION
       - HBASE_PLUGIN_VERSION

dev-support/ranger-docker/docker-compose.ranger-hive.yml

@@ -26,6 +26,12 @@ services:
         condition: service_started
       ranger-hadoop:
         condition: service_healthy
+    healthcheck:
+      test: [ "CMD-SHELL", "su hdfs -c \"/opt/hadoop/bin/hdfs dfsadmin -report | grep -q 'Live datanodes'\"" ]
+      interval: 30s
+      timeout: 10s
+      retries: 30
+      start_period: 40s
     environment:
       - HIVE_HADOOP_VERSION
       - HIVE_VERSION

dev-support/ranger-docker/docker-compose.ranger-kafka.yml

@@ -22,9 +22,18 @@ services:
         condition: service_started
       ranger-zk:
         condition: service_started
+    healthcheck:
+      test: [ "CMD-SHELL", "echo 'ping' | kafka-console-producer.sh --broker-list localhost:9092 --topic health-check-topic && kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic health-check-topic --from-beginning --max-messages 1 | grep 'ping'" ]
+      interval: 60s
+      timeout: 20s
+      retries: 5
+      start_period: 60s
     environment:
-      - KAFKA_VERSION
-      - KAFKA_PLUGIN_VERSION
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://ranger-kafka:9092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_VERSION: ${KAFKA_VERSION}
+      KAFKA_PLUGIN_VERSION: ${KAFKA_PLUGIN_VERSION}
 
 networks:
   ranger:

dev-support/ranger-docker/docker-compose.ranger-kms.yml

@@ -20,6 +20,12 @@ services:
     depends_on:
       ranger:
         condition: service_started
+    healthcheck:
+      test: [ "CMD-SHELL", "grep 'Ranger KMS Started' /var/log/ranger/kms/ranger-kms*.log" ]
+      interval: 40s
+      timeout: 5s
+      retries: 10
+      start_period: 40s
     environment:
       - KMS_VERSION
       - RANGER_DB_TYPE

dev-support/ranger-docker/docker-compose.ranger-knox.yml

@@ -22,6 +22,12 @@ services:
         condition: service_started
       ranger-zk:
         condition: service_started
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -k -i -u admin:admin-password 'https://localhost:8443/gateway/sandbox/webhdfs/v1/?op=LISTSTATUS'" ]
+      interval: 60s
+      timeout: 10s
+      retries: 10
+      start_period: 10s
     environment:
       - KNOX_VERSION
       - KNOX_PLUGIN_VERSION

dev-support/ranger-docker/docker-compose.ranger-ozone.yml

@@ -38,6 +38,12 @@ services:
         condition: service_started
       ranger-solr:
         condition: service_started
+    healthcheck:
+      test: [ "CMD-SHELL", "su om -c \"/opt/hadoop/bin/ozone fs -ls ofs://om/\"" ]
+      interval: 40s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
     environment:
       ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
       OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false

dev-support/ranger-docker/docker-compose.ranger-tagsync.yml

@@ -17,6 +17,12 @@ services:
     depends_on:
       ranger:
         condition: service_started
+    healthcheck:
+      test: [ "CMD-SHELL", "grep 'Initializing TAG source and sink' /var/log/ranger/tagsync/tagsync*log*" ]
+      interval: 30s
+      timeout: 3s
+      retries: 10
+      start_period: 40s
     environment:
       - TAGSYNC_VERSION
       - DEBUG_TAGSYNC=${DEBUG_TAGSYNC:-false}

dev-support/ranger-docker/docker-compose.ranger-usersync.yml

@@ -17,6 +17,12 @@ services:
     depends_on:
       ranger:
         condition: service_started
+    healthcheck:
+      test: [ "CMD-SHELL", "grep 'uploaded to ranger admin' /var/log/ranger/usersync/usersync*.log*" ]
+      interval: 30s
+      timeout: 3s
+      retries: 10
+      start_period: 40s
     environment:
       - USERSYNC_VERSION
       - ENABLE_FILE_SYNC_SOURCE