RANGER-5310:Include Apache Tez as the process framework for ranger-hive docker

dev-support/ranger-docker/.env

@@ -11,46 +11,66 @@ RANGER_BASE_IMAGE=apache/ranger-base
 RANGER_BASE_VERSION=20250707-1-8
 # Java version used to build Apache Ranger is present as suffix: -8, valid values for suffix: -8, -11, -17
 RANGER_BASE_BUILD_VERSION=20250707-1-8
+RANGER_VERSION=3.0.0-SNAPSHOT
 
-# third party image versions
-MARIADB_VERSION=10.7.3
-POSTGRES_VERSION=12
-ORACLE_VERSION=23.6
-SQLSERVER_VERSION=2019-latest
-ENABLE_DB_MOUNT=true
-ZK_VERSION=3.9.2
-SOLR_VERSION=8.11.3
-
-# service versions
+# Hadoop Configuration
 HADOOP_VERSION=3.3.6
-HBASE_VERSION=2.6.0
+HDFS_PLUGIN_VERSION=3.0.0-SNAPSHOT
+YARN_PLUGIN_VERSION=3.0.0-SNAPSHOT
+
+# Hive Configuration
+HIVE_HADOOP_VERSION=3.3.6
 HIVE_VERSION=4.0.1
-HIVE_HADOOP_VERSION=3.1.1
+HIVE_PLUGIN_VERSION=3.0.0-SNAPSHOT
+
+# Tez Configuration
+TEZ_VERSION=0.10.4
+
+# HBase Configuration
+HBASE_VERSION=2.6.0
+HBASE_PLUGIN_VERSION=3.0.0-SNAPSHOT
+
+# Kafka Configuration
 KAFKA_VERSION=2.8.2
+KAFKA_PLUGIN_VERSION=3.0.0-SNAPSHOT
+
+# Knox Configuration
 KNOX_VERSION=2.0.0
-TRINO_VERSION=377
-OZONE_VERSION=1.4.0
-OZONE_RUNNER_VERSION=20230615-1
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_OPTS=
+KNOX_PLUGIN_VERSION=3.0.0-SNAPSHOT
 
-# versions of ranger services
-RANGER_VERSION=3.0.0-SNAPSHOT
+# KMS Configuration
 KMS_VERSION=3.0.0-SNAPSHOT
+
+# Usersync Configuration
 USERSYNC_VERSION=3.0.0-SNAPSHOT
+
+# Tagsync Configuration
 TAGSYNC_VERSION=3.0.0-SNAPSHOT
 
-# plugin versions
-HDFS_PLUGIN_VERSION=3.0.0-SNAPSHOT
-YARN_PLUGIN_VERSION=3.0.0-SNAPSHOT
-HIVE_PLUGIN_VERSION=3.0.0-SNAPSHOT
-HBASE_PLUGIN_VERSION=3.0.0-SNAPSHOT
-KAFKA_PLUGIN_VERSION=3.0.0-SNAPSHOT
-KNOX_PLUGIN_VERSION=3.0.0-SNAPSHOT
-TRINO_PLUGIN_VERSION=3.0.0-SNAPSHOT
+# Solr Configuration
+SOLR_VERSION=8.11.2
+
+# Zookeeper Configuration
+ZK_VERSION=3.8.4
+
+# Database Versions
+POSTGRES_VERSION=13.16
+MARIADB_VERSION=10.6
+ORACLE_VERSION=21.3.0-xe
+SQLSERVER_VERSION=2022-latest
+
+# Ozone Configuration
+OZONE_VERSION=1.4.0
 OZONE_PLUGIN_VERSION=3.0.0-SNAPSHOT
+OZONE_RUNNER_IMAGE=apache/ozone-runner
+OZONE_RUNNER_VERSION=20240625
+
+# Trino Configuration
+TRINO_VERSION=435
+TRINO_PLUGIN_VERSION=3.0.0-SNAPSHOT
 
-# To enable debug logs
+# Debug Configuration
 DEBUG_ADMIN=false
 DEBUG_USERSYNC=false
 DEBUG_TAGSYNC=false
+ENABLE_FILE_SYNC_SOURCE=false

dev-support/ranger-docker/Dockerfile.ranger

@@ -23,12 +23,20 @@ ARG RANGER_VERSION
 ARG RANGER_DB_TYPE
 ARG TARGETARCH
 
+# Set Ranger environment variables
+ENV RANGER_HOME=/opt/ranger
+ENV RANGER_SCRIPTS=/home/ranger/scripts
+
 COPY ./dist/version                               /home/ranger/dist/
 COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/
 
 COPY ./scripts/ranger.sh                                         ${RANGER_SCRIPTS}/
 COPY ./scripts/ranger-admin-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-admin-install.properties
 COPY ./scripts/create-ranger-services.py                         ${RANGER_SCRIPTS}/
+COPY ./scripts/create-users.sh                                   ${RANGER_SCRIPTS}/
+
+# Switch to root to create system directories
+USER root
 
 RUN    tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} \
     && ln -s ${RANGER_HOME}/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}/admin \
@@ -37,6 +45,8 @@ RUN    tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --direct
     && mkdir -p /var/run/ranger \
     && mkdir -p /var/log/ranger \
     && chown -R ranger:ranger ${RANGER_HOME}/admin/ ${RANGER_SCRIPTS}/ /var/run/ranger/ /var/log/ranger/ \
+    && chmod +x ${RANGER_SCRIPTS}/create-users.sh \
+    && ${RANGER_SCRIPTS}/create-users.sh \
     && chmod 755 ${RANGER_SCRIPTS}/ranger.sh \
     && mkdir -p /usr/share/java/
 

dev-support/ranger-docker/Dockerfile.ranger-hadoop

@@ -14,29 +14,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG RANGER_BASE_IMAGE
-ARG RANGER_BASE_VERSION
-FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+FROM ranger
 
 ARG HADOOP_VERSION
 ARG HDFS_PLUGIN_VERSION
 ARG YARN_PLUGIN_VERSION
+ARG TEZ_VERSION
 
 
 COPY ./dist/version                                          /home/ranger/dist/
 COPY ./dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz /home/ranger/dist/
 COPY ./dist/ranger-${YARN_PLUGIN_VERSION}-yarn-plugin.tar.gz /home/ranger/dist/
 COPY ./downloads/hadoop-${HADOOP_VERSION}.tar.gz             /home/ranger/dist/
+COPY ./downloads/apache-tez-${TEZ_VERSION}-bin.tar.gz        /home/ranger/dist/
 
 COPY ./scripts/ranger-hadoop-setup.sh                   /home/ranger/scripts/
 COPY ./scripts/ranger-hadoop.sh                         /home/ranger/scripts/
 COPY ./scripts/ranger-hadoop-mkdir.sh                   /home/ranger/scripts/
 COPY ./scripts/ranger-hdfs-plugin-install.properties    /home/ranger/scripts/
 COPY ./scripts/ranger-yarn-plugin-install.properties    /home/ranger/scripts/
 
+# Switch to root to create system directories
+USER root
+
 RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ && \
     ln -s /opt/hadoop-${HADOOP_VERSION} /opt/hadoop && \
     rm -f /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz && \
+    tar xvfz /home/ranger/dist/apache-tez-${TEZ_VERSION}-bin.tar.gz --directory=/opt/ && \
+    ln -s /opt/apache-tez-${TEZ_VERSION}-bin /opt/tez && \
     tar xvfz /home/ranger/dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz --directory=/opt/ranger && \
     ln -s /opt/ranger/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin /opt/ranger/ranger-hdfs-plugin && \
     rm -f /home/ranger/dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz && \

dev-support/ranger-docker/Dockerfile.ranger-hbase

@@ -14,9 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG RANGER_BASE_IMAGE
-ARG RANGER_BASE_VERSION
-FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+FROM ranger
 
 ARG HBASE_VERSION
 ARG HBASE_PLUGIN_VERSION
@@ -31,6 +29,9 @@ COPY ./scripts/ranger-hbase.sh                           /home/ranger/scripts/
 COPY ./scripts/ranger-hbase-plugin-install.properties    /home/ranger/scripts/
 COPY ./scripts/hbase-site.xml                            /home/ranger/scripts/
 
+# Switch to root to create system directories
+USER root
+
 RUN tar xvfz /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \
     ln -s /opt/hbase-${HBASE_VERSION} /opt/hbase && \
     rm -f /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz && \
@@ -48,4 +49,7 @@ RUN apt-get update && \
 ENV HBASE_HOME=/opt/hbase
 ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hbase/bin
 
+# Ensure container runs as root for SSH and service management
+USER root
+
 ENTRYPOINT [ "/home/ranger/scripts/ranger-hbase.sh" ]

dev-support/ranger-docker/Dockerfile.ranger-hive

@@ -13,22 +13,22 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG RANGER_DB_TYPE
+# Only the ARGs needed for this build stage
 
-ARG RANGER_BASE_IMAGE
-ARG RANGER_BASE_VERSION
-FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+FROM ranger
 
 ARG HIVE_VERSION
 ARG HIVE_HADOOP_VERSION
 ARG HIVE_PLUGIN_VERSION
+ARG TEZ_VERSION
 ARG RANGER_DB_TYPE
 
 
 COPY ./dist/version                                          /home/ranger/dist/
 COPY ./dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz /home/ranger/dist/
 COPY ./downloads/apache-hive-${HIVE_VERSION}-bin.tar.gz      /home/ranger/dist/
 COPY ./downloads/hadoop-${HIVE_HADOOP_VERSION}.tar.gz        /home/ranger/dist/
+COPY ./downloads/apache-tez-${TEZ_VERSION}-bin.tar.gz        /home/ranger/dist/
 COPY ./downloads/postgresql-42.2.16.jre7.jar                 /home/ranger/dist/
 COPY ./downloads/mysql-connector-java-8.0.28.jar             /home/ranger/dist/
 COPY ./downloads/ojdbc8.jar                                  /home/ranger/dist/
@@ -38,7 +38,10 @@ COPY ./scripts/ranger-hive.sh                           /home/ranger/scripts/
 COPY ./scripts/ranger-hive-plugin-install.properties    /home/ranger/scripts/
 COPY ./scripts/hive-site-${RANGER_DB_TYPE}.xml          /home/ranger/scripts/hive-site.xml
 
-RUN tar xvfz /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz --directory=/opt/ && \
+# Switch to root to create system directories
+USER root
+
+RUN cd /opt && tar xzf /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
     ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \
     rm -f /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
     mv /home/ranger/dist/postgresql-42.2.16.jre7.jar /opt/hive/lib/ && \
@@ -47,6 +50,9 @@ RUN tar xvfz /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz --director
     tar xvfz /home/ranger/dist/hadoop-${HIVE_HADOOP_VERSION}.tar.gz --directory=/opt/ && \
     ln -s /opt/hadoop-${HIVE_HADOOP_VERSION} /opt/hadoop && \
     rm -f /home/ranger/dist/hadoop-${HIVE_HADOOP_VERSION}.tar.gz && \
+    tar xvfz /home/ranger/dist/apache-tez-${TEZ_VERSION}-bin.tar.gz --directory=/opt/ && \
+    ln -s /opt/apache-tez-${TEZ_VERSION}-bin /opt/tez && \
+    rm -f /home/ranger/dist/apache-tez-${TEZ_VERSION}-bin.tar.gz && \
     tar xvfz /home/ranger/dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz --directory=/opt/ranger && \
     ln -s /opt/ranger/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin /opt/ranger/ranger-hive-plugin && \
     rm -f /home/ranger/dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz && \
@@ -55,7 +61,11 @@ RUN tar xvfz /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz --director
 
 ENV HIVE_HOME=/opt/hive
 ENV HADOOP_HOME=/opt/hadoop
-ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin
+ENV TEZ_HOME=/opt/tez
+ENV TEZ_CONF_DIR=/opt/tez/conf
+ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/hadoop/bin:/opt/tez/bin
 
+# Ensure container runs as root for SSH and service management
+USER root
 
 ENTRYPOINT [ "/home/ranger/scripts/ranger-hive.sh" ]

dev-support/ranger-docker/Dockerfile.ranger-kafka

@@ -14,9 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG RANGER_BASE_IMAGE
-ARG RANGER_BASE_VERSION
-FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+FROM ranger
 
 ARG KAFKA_VERSION
 ARG KAFKA_PLUGIN_VERSION
@@ -30,6 +28,9 @@ COPY ./scripts/ranger-kafka-setup.sh                     /home/ranger/scripts/
 COPY ./scripts/ranger-kafka.sh                           /home/ranger/scripts/
 COPY ./scripts/ranger-kafka-plugin-install.properties    /home/ranger/scripts/
 
+# Switch to root to create system directories
+USER root
+
 RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
     ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \
     rm -f /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz && \
@@ -39,7 +40,11 @@ RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/
     cp -f /home/ranger/scripts/ranger-kafka-plugin-install.properties /opt/ranger/ranger-kafka-plugin/install.properties && \
     chmod 744 ${RANGER_SCRIPTS}/ranger-kafka-setup.sh ${RANGER_SCRIPTS}/ranger-kafka.sh
 
+
 ENV KAFKA_HOME=/opt/kafka
 ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin
 
+# Ensure container runs as root for SSH and service management
+USER root
+
 ENTRYPOINT [ "/home/ranger/scripts/ranger-kafka.sh" ]

dev-support/ranger-docker/Dockerfile.ranger-kms

@@ -13,10 +13,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG RANGER_DB_TYPE
-ARG RANGER_BASE_IMAGE
-ARG RANGER_BASE_VERSION
-FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} AS ranger-kms
+
+FROM ranger AS ranger-kms
 
 ARG KMS_VERSION
 ARG RANGER_DB_TYPE
@@ -27,6 +25,9 @@ COPY ./dist/ranger-${KMS_VERSION}-kms.tar.gz                   /home/ranger/dist
 COPY ./scripts/ranger-kms.sh                                   ${RANGER_SCRIPTS}/
 COPY ./scripts/ranger-kms-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-kms-install.properties
 
+# Switch to root to create system directories
+USER root
+
 RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RANGER_HOME} && \
     ln -s ${RANGER_HOME}/ranger-${KMS_VERSION}-kms ${RANGER_HOME}/kms && \
     rm -f /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz && \
@@ -38,7 +39,7 @@ RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RA
     ln -s /etc/init.d/ranger-kms /etc/rc3.d/S88ranger-kms && \
     ln -s /etc/init.d/ranger-kms /etc/rc3.d/K90ranger-kms && \
     ln -s ${RANGER_HOME}/kms/ranger-kms-services.sh /usr/bin/ranger-kms-services.sh && \
-    chown -R rangerkms:ranger ${RANGER_HOME}/kms/ ${RANGER_SCRIPTS}/ /var/run/ranger_kms/ /var/log/ranger/ && \
+    chown -R rangerkms:ranger ${RANGER_HOME}/kms/ ${RANGER_SCRIPTS}/ /var/run/ranger_kms/ /var/log/ranger/ /etc/ranger && \
     chmod 744 ${RANGER_SCRIPTS}/ranger-kms.sh
 
 FROM ranger-kms AS ranger_postgres

dev-support/ranger-docker/Dockerfile.ranger-knox

@@ -14,9 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG RANGER_BASE_IMAGE
-ARG RANGER_BASE_VERSION
-FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+FROM ranger
 
 ARG KNOX_VERSION
 ARG KNOX_PLUGIN_VERSION
@@ -32,6 +30,9 @@ COPY ./scripts/ranger-knox-plugin-install.properties    /home/ranger/scripts/
 COPY ./scripts/ranger-knox-expect.py                    /home/ranger/scripts/
 COPY ./scripts/ranger-knox-sandbox.xml                  /home/ranger/scripts/
 
+# Switch to root to create system directories
+USER root
+
 RUN tar xvfz /home/ranger/dist/knox-${KNOX_VERSION}.tar.gz --directory=/opt/ && \
     ln -s /opt/knox-${KNOX_VERSION} /opt/knox && \
     rm -f /home/ranger/dist/knox-${KNOX_VERSION}.tar.gz && \
@@ -50,4 +51,7 @@ RUN chmod a+rwx /home/ranger/scripts/ranger-knox-expect.py
 RUN pip3 install pexpect
 RUN python3 /home/ranger/scripts/ranger-knox-expect.py
 
+# Ensure container runs as root for SSH and service management
+USER root
+
 ENTRYPOINT [ "/home/ranger/scripts/ranger-knox.sh" ]

dev-support/ranger-docker/Dockerfile.ranger-tagsync

@@ -14,9 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG RANGER_BASE_IMAGE
-ARG RANGER_BASE_VERSION
-FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+FROM ranger
 
 ARG TAGSYNC_VERSION
 
@@ -27,6 +25,9 @@ COPY ./scripts/ranger-tagsync.sh                     ${RANGER_SCRIPTS}/
 COPY ./scripts/ranger-tagsync-install.properties     ${RANGER_SCRIPTS}/
 COPY ./scripts/ranger-tagsync-tags.json              ${RANGER_SCRIPTS}/
 
+# Switch to root to create system directories
+USER root
+
 RUN tar xvfz /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz --directory=${RANGER_HOME} && \
     ln -s ${RANGER_HOME}/ranger-${TAGSYNC_VERSION}-tagsync ${RANGER_HOME}/tagsync && \
     rm -f /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz && \