Merge branch 'migrate-rdbms' into devel

builder/requirements.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2023 Cloudera, Inc. All Rights Reserved.
+# Copyright 2024 Cloudera, Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,11 +16,3 @@
 collections:
   - source: .
     type: dir
-
-roles:
-  - name: geerlingguy.postgresql
-    version: 2.2.0
-
-  # geerlingguy.mysql with fix for issue #332
-  - src: https://github.com/dbeech/ansible-role-mysql
-    version: master

galaxy.yml

@@ -16,7 +16,7 @@
 
 namespace:  cloudera
 name:       cluster
-version:    4.3.0
+version:    4.3.0-dev
 readme:     README.md
 authors:    []
 
@@ -35,7 +35,7 @@ tags:
 
 dependencies:
   'ansible.posix':      '1.3.0'
-  'community.crypto':   '2.2.1'
+  'community.crypto':   '2.17.1'
   'community.general':  '4.5.0'
 
 repository:     https://github.com/cloudera-labs/cloudera.cluster

requirements.yml

@@ -1,6 +1,6 @@
 ---
 
-# Copyright 2023 Cloudera, Inc.
+# Copyright 2024 Cloudera, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,24 +14,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-roles:
-  - name: geerlingguy.postgresql
-    version: 2.2.0
-
-  # geerlingguy.mysql with fix for issue #332
-  - src: https://github.com/dbeech/ansible-role-mysql
-    version: master
 
 collections:
   - name: ansible.posix
     version: 1.3.0
   - name: community.crypto
-    version: 2.2.1
+    version: 2.17.1
   - name: community.general
     version: 4.5.0
   - name: community.mysql
-    version: 3.1.0
+    version: 3.8.0
   - name: community.postgresql
-    version: 1.6.1
+    version: 3.3.0
   - name: freeipa.ansible_freeipa
     version: 1.11.1

roles/config/cluster/base/templates/configs/tls-7.1.0.j2

@@ -109,7 +109,7 @@ LIVY:
     ssl_server_keystore_location: {{ tls_keystore_path_generic }}
     ssl_server_keystore_password: {{ tls_keystore_password }}
 OZONE:
-  DATANODE:
+  OZONE_DATANODE:
     ssl_client_truststore_location: {{ tls_truststore_path }}
     ssl_client_truststore_password: {{ tls_truststore_password }}
     ssl_enabled: true

roles/config/cluster/base/templates/configs/tls-7.1.7.j2

@@ -0,0 +1,4 @@
+---
+OZONE:
+  SERVICEWIDE:
+    hdds.grpc.tls.enabled: true

roles/config/cluster/base/templates/configs/tls-cm-7.j2

@@ -104,8 +104,7 @@ LIVY:
     ssl_enabled: true
     ssl_server_keystore_location: {{ tls_keystore_path_generic }}
     ssl_server_keystore_password: {{ tls_keystore_password }}
-OZONE:
-  DATANODE:
+  OZONE_DATANODE:
     ssl_client_truststore_location: {{ tls_truststore_path }}
     ssl_client_truststore_password: {{ tls_truststore_password }}
     ssl_enabled: true
@@ -143,6 +142,13 @@ OZONE:
     ssl_server_keystore_keypassword: {{ tls_keystore_password }}
     ssl_server_keystore_location: {{ tls_keystore_path_generic }}
     ssl_server_keystore_password: {{ tls_keystore_password }}
+  HTTPFS_GATEWAY:
+    ssl_client_truststore_location: {{ tls_truststore_path }}
+    ssl_client_truststore_password: {{ tls_truststore_password }}
+    ssl_enabled: true
+    ssl_server_keystore_keypassword: {{ tls_keystore_password }}
+    ssl_server_keystore_location: {{ tls_keystore_path_generic }}
+    ssl_server_keystore_password: {{ tls_keystore_password }}
 NIFI:
   NIFI_NODE:
     ssl_enabled: true

roles/config/cluster/base/vars/main.yml

@@ -64,10 +64,12 @@ custom_config_templates:
     condition: "{{ cluster.security.tls | default(False) and cloudera_runtime_version is version('7.1.0','>=') }}"
   - template: configs/tls-7.1.4.j2
     condition: "{{ cluster.security.tls | default(False) and cloudera_runtime_version is version('7.1.4','>=') and (cloudera_runtime_pre_upgrade is undefined or cloudera_runtime_pre_upgrade is version('7.1.4','>=')) }}"
+  - template: configs/tls-7.1.7.j2
+    condition: "{{ cluster.security.tls | default(False) and cloudera_runtime_version is version('7.1.7','>=') }}"
   - template: configs/tls-7.3.1.j2
     condition: "{{ cluster.security.tls | default(False) and cloudera_manager_version is version('7.3.1', '>=') }}"
   - template: configs/tls-cm-7.j2
-    condition: "{{ cluster.security.tls | default(False) and cloudera_manager_version is version('7.1.0','>=') }}"
+    condition: "{{ cluster.security.tls | default(False) and cloudera_manager_version is version('7.1.0','>=') }}"  
 # Custom configurations for Cloudera Streams Processing components on CDH 6.x
   - template: configs/schemaregistry.j2
     condition: >-

roles/config/cluster/ecs/tasks/main.yml

@@ -17,8 +17,9 @@
 # This variable is used by other roles
 # please take care when changing it
 - set_fact:
-    databases: "{{ database_defaults | combine(cluster.databases | default({}), recursive=True) }}"
-
+#    databases: "{{ database_defaults | combine(cluster.databases | default({}), recursive=True) }}"
+    databases: []
+
 - name: Retrieve repository metadata
   include_role:
     name: cloudera.cluster.deployment.repometa

roles/deployment/definition/defaults/main.yml

@@ -1,4 +1,4 @@
-# Copyright 2023 Cloudera, Inc.
+# Copyright 2024 Cloudera, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,12 +24,15 @@ default_database_versions:
   postgresql:
     '7': 10
     '8': 12
+    '9': 14
   mariadb:
     '7': 10.2
-    '8': 10.2
+    '8': 10.6
+    '9': 10.6
   mysql:
     '7': 5.7
-    '8': 8.0  
+    '8': 8.0 
+    '9': 8.0 
 
 # Located in cloudera.cluster.infrastructure.krb5_common
 #krb5_realm: CLOUDERA.LOCAL
@@ -125,7 +128,7 @@ database_defaults:
   QUEUEMANAGER:
     host: "{{ database_host }}"
     port: "{{ database_type | cloudera.cluster.default_database_port }}"
-    type: postgresql
+    type: "{{ database_type }}"
     name: queuemanager
     user: queuemanager
     password: "{{ database_default_password }}"

roles/infrastructure/krb5_client/tasks/freeipa.yml

@@ -67,3 +67,12 @@
   when:
     - krb5_kdc_type == 'Red Hat IPA'
     - "'cluster' in group_names or 'cloudera_manager' in group_names"
+
+- name: Remove ipa includedir directive
+  lineinfile:
+    path: /etc/krb5.conf
+    regexp: "^includedir /etc/krb5.conf.d/"
+    state: absent
+  when:
+    - krb5_kdc_type == 'Red Hat IPA'
+    - "'cluster' in group_names or 'cloudera_manager' in group_names"