Skip to content

Commit 3e0ee50

Browse files
baojun-zhangbaojun-zhang
committed
fix(hadoop,hbase): add lifecycle mount && fix big number format error && inject namenode pod fqdn via container env
1 parent e6fcef0 commit 3e0ee50

5 files changed

Lines changed: 32 additions & 4 deletions

File tree

‎addons-cluster/hadoop/templates/cluster.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@
130130
HDFS_PERMISSIONS_SUPERUSERGROUP: {{ .Values.hdfs.permissions.superusergroup | quote }}
131131
HDFS_REGISTRATION_IP_HOSTNAME_CHECK: {{ .Values.hdfs.registration.ipHostnameCheck | quote }}
132132
HDFS_NAMENODE_WEBHDFS_ENABLED: {{ .Values.hdfs.namenode.webhdfsEnabled | quote }}
133-
HDFS_NAMENODE_RESOURCE_DU_RESERVED: {{ .Values.hdfs.namenode.resourceDuReserved | quote }}
133+
HDFS_NAMENODE_RESOURCE_DU_RESERVED: {{ printf "%.0f" (float64 .Values.hdfs.namenode.resourceDuReserved) | quote }}
134134
HDFS_CLIENT_RETRY_POLICY_ENABLED: {{ .Values.hdfs.client.retryPolicyEnabled | quote }}
135135
HDFS_CLIENT_RETRY_POLICY_SPEC: {{ .Values.hdfs.client.retryPolicySpec | quote }}
136136
{{- if $isCluster }}
@@ -216,7 +216,7 @@
216216
HDFS_NAMENODE_HTTP_PORT: {{ .Values.hdfs.ports.namenode.http | quote }}
217217
HDFS_DATANODE_HANDLER_COUNT: {{ .Values.hdfs.datanode.handlerCount | quote }}
218218
HDFS_DATANODE_FAILED_VOLUMES_TOLERATED: {{ .Values.hdfs.datanode.failedVolumesTolerated | quote }}
219-
HDFS_DATANODE_DU_RESERVED: {{ .Values.hdfs.datanode.duReserved | quote }}
219+
HDFS_DATANODE_DU_RESERVED: {{ printf "%.0f" (float64 .Values.hdfs.datanode.duReserved) | quote }}
220220
HDFS_PERMISSIONS_ENABLED: {{ .Values.hdfs.permissions.enabled | quote }}
221221
HDFS_HA_NAMENODE_IDS: {{ .Values.hdfs.ha.nameNodeIds | quote }}
222222
HDFS_CLIENT_RETRY_POLICY_ENABLED: {{ .Values.hdfs.client.retryPolicyEnabled | quote }}

‎addons-cluster/hbase/templates/cluster.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,8 @@
217217
HBASE_REGIONSERVER_HANDLER_COUNT: {{ .Values.hbase.standalone.regionserverHandlerCount | quote }}
218218
HBASE_HSTORE_FLUSHER_COUNT: {{ .Values.hbase.standalone.hstoreFlusherCount | quote }}
219219
HBASE_CLIENT_SCANNER_CACHING: {{ .Values.hbase.common.clientScannerCaching | quote }}
220-
HBASE_HREGION_MAX_FILESIZE: {{ .Values.hbase.standalone.regionMaxFileSize | quote }}
221-
HBASE_HREGION_MEMSTORE_FLUSH_SIZE: {{ .Values.hbase.common.regionMemstoreFlushSize | quote }}
220+
HBASE_HREGION_MAX_FILESIZE: {{ printf "%.0f" (float64 .Values.hbase.standalone.regionMaxFileSize) | quote }}
221+
HBASE_HREGION_MEMSTORE_FLUSH_SIZE: {{ printf "%.0f" (float64 .Values.hbase.common.regionMemstoreFlushSize) | quote }}
222222
HBASE_PROCEDURE_STORE_WAL_USE_HSYNC: {{ .Values.hbase.standalone.procedureStoreWalUseHsync | quote }}
223223
HBASE_REPLICATION_ENABLED: {{ .Values.hbase.standalone.replicationEnabled | quote }}
224224
HBASE_SECURITY_AUTHENTICATION: {{ .Values.hbase.security.authentication | quote }}

‎addons/hadoop/templates/cmpd-hdfs-namenode-standalone.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,10 @@ spec:
103103
valueFrom:
104104
fieldRef:
105105
fieldPath: metadata.name
106+
- name: POD_FQDN
107+
valueFrom:
108+
fieldRef:
109+
fieldPath: metadata.annotations['kubeblocks.io/pod-fqdn']
106110
- name: HADOOP_HOME
107111
value: {{ .Values.runtime.hadoopHome }}
108112
- name: HADOOP_CONF_DIR
@@ -167,6 +171,8 @@ spec:
167171
- name: namenode-config
168172
mountPath: {{ .Values.runtime.configDir }}/hdfs-site.xml
169173
subPath: hdfs-site.xml
174+
- name: lifecycle
175+
mountPath: /lifecycle
170176
- name: hadoop-logs
171177
mountPath: {{ .Values.runtime.hadoopLogDir }}
172178
- name: scripts
@@ -208,6 +214,8 @@ spec:
208214
volumes:
209215
- name: hadoop-common
210216
emptyDir: {}
217+
- name: lifecycle
218+
emptyDir: {}
211219
- name: hadoop-logs
212220
emptyDir: {}
213221
updateStrategy: {{ .Values.updateStrategy }}

‎addons/hadoop/templates/cmpd-hdfs-namenode.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,10 @@ spec:
145145
valueFrom:
146146
fieldRef:
147147
fieldPath: metadata.name
148+
- name: POD_FQDN
149+
valueFrom:
150+
fieldRef:
151+
fieldPath: metadata.annotations['kubeblocks.io/pod-fqdn']
148152
- name: HADOOP_HOME
149153
value: {{ .Values.runtime.hadoopHome }}
150154
- name: HADOOP_CONF_DIR

‎hack/verify-hbase-hadoop-render.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,10 @@ helm template test addons-cluster/hbase \
129129
--set serviceRefs.hdfsNamenode.clusterServiceSelector.cluster=external-hdfs-cluster \
130130
> "${TMP_DIR}/hbase-cluster-serviceref-default-ns.yaml"
131131

132+
helm template test addons-cluster/hbase \
133+
--set topology=standalone \
134+
> "${TMP_DIR}/hbase-cluster-standalone.yaml"
135+
132136
assert_contains "${TMP_DIR}/hadoop-addon.yaml" "kind: ParamConfigRenderer"
133137
assert_contains "${TMP_DIR}/hadoop-addon.yaml" "name: hdfs-namenode-config-renderer"
134138
assert_contains "${TMP_DIR}/hadoop-addon.yaml" "refresh-decommission-state.sh: |-"
@@ -146,6 +150,10 @@ assert_contains "${TMP_DIR}/hadoop-cluster.yaml" 'name: test-hdfs-decommission-s
146150
assert_contains "${TMP_DIR}/hadoop-cluster.yaml" 'name: test-hdfs-decommission-state-editor'
147151
assert_contains "${TMP_DIR}/hadoop-cluster-custom-state.yaml" 'name: custom-decommission-state'
148152
assert_contains "${TMP_DIR}/hadoop-cluster-custom-state.yaml" 'name: custom-decommission-state-editor'
153+
assert_contains "${TMP_DIR}/hadoop-cluster.yaml" 'HDFS_NAMENODE_RESOURCE_DU_RESERVED: "1073741824"'
154+
assert_contains "${TMP_DIR}/hadoop-cluster.yaml" 'HDFS_DATANODE_DU_RESERVED: "1073741824"'
155+
assert_not_contains "${TMP_DIR}/hadoop-cluster.yaml" 'HDFS_NAMENODE_RESOURCE_DU_RESERVED: "1.073741824e+09"'
156+
assert_not_contains "${TMP_DIR}/hadoop-cluster.yaml" 'HDFS_DATANODE_DU_RESERVED: "1.073741824e+09"'
149157

150158
assert_contains "${TMP_DIR}/hbase-addon.yaml" "check-hmaster-live.sh: |-"
151159
assert_contains "${TMP_DIR}/hbase-addon.yaml" "check-hmaster-ready.sh: |-"
@@ -163,6 +171,10 @@ assert_contains "${TMP_DIR}/hbase-cluster-fallback.yaml" 'HDFS_NAMENODE_HTTP_POR
163171
assert_not_contains "${TMP_DIR}/hbase-cluster-fallback.yaml" "name: hdfs-namenode"
164172
assert_contains "${TMP_DIR}/hbase-cluster-serviceref.yaml" "name: hdfs-namenode"
165173
assert_contains "${TMP_DIR}/hbase-cluster-serviceref-default-ns.yaml" 'HDFS_NAMESERVICE: "external-ns"'
174+
assert_contains "${TMP_DIR}/hbase-cluster-standalone.yaml" 'HBASE_HREGION_MAX_FILESIZE: "10737418240"'
175+
assert_contains "${TMP_DIR}/hbase-cluster-standalone.yaml" 'HBASE_HREGION_MEMSTORE_FLUSH_SIZE: "134217728"'
176+
assert_not_contains "${TMP_DIR}/hbase-cluster-standalone.yaml" 'HBASE_HREGION_MAX_FILESIZE: "1.073741824e+10"'
177+
assert_not_contains "${TMP_DIR}/hbase-cluster-standalone.yaml" 'HBASE_HREGION_MEMSTORE_FLUSH_SIZE: "1.34217728e+08"'
166178

167179
assert_contains "${ROOT_DIR}/addons/hadoop/scripts/check-name-status.sh" '_NN_HTTP_PORT="${HDFS_NAMENODE_HTTP_PORT:-9870}"'
168180
assert_contains "${ROOT_DIR}/addons/hadoop/scripts/check-journal-status.sh" '_PORTS="${HDFS_JOURNALNODE_HTTP_PORT:-8480}"'
@@ -228,6 +240,8 @@ assert_not_contains "${ROOT_DIR}/addons/hadoop/templates/cmpd-hdfs-datanode-stan
228240
assert_contains "${ROOT_DIR}/addons/hadoop/templates/cmpd-hdfs-datanode.yaml" 'HDFS_DECOMMISSION_DYNAMIC_EXCLUDE_FILE'
229241
assert_contains "${ROOT_DIR}/addons/hadoop/templates/cmpd-hdfs-namenode.yaml" 'HDFS_DECOMMISSION_DYNAMIC_EXCLUDE_FILE'
230242
assert_contains "${ROOT_DIR}/addons/hadoop/templates/cmpd-hdfs-namenode-standalone.yaml" 'HDFS_DECOMMISSION_DYNAMIC_EXCLUDE_FILE'
243+
assert_contains "${ROOT_DIR}/addons/hadoop/templates/cmpd-hdfs-namenode-standalone.yaml" 'name: lifecycle'
244+
assert_contains "${ROOT_DIR}/addons/hadoop/templates/cmpd-hdfs-namenode-standalone.yaml" 'mountPath: /lifecycle'
231245
assert_contains "${ROOT_DIR}/addons-cluster/hadoop/templates/decommission-state-configmap.yaml" 'stateConfigMapName'
232246
assert_contains "${ROOT_DIR}/addons-cluster/hadoop/templates/decommission-rbac.yaml" 'stateConfigMapName'
233247
assert_contains "${ROOT_DIR}/addons-cluster/hadoop/templates/decommission-rbac.yaml" ' - create'
@@ -242,6 +256,8 @@ assert_contains "${ROOT_DIR}/addons/hbase/templates/cmpd-hmaster.yaml" 'name: HD
242256
assert_contains "${ROOT_DIR}/addons/hbase/templates/cmpd-hmaster.yaml" 'name: HDFS_NAMENODE_POD_FQDNS'
243257
assert_contains "${ROOT_DIR}/addons/hbase/templates/cmpd-hmaster.yaml" 'podFQDNs: Optional'
244258
assert_contains "${ROOT_DIR}/addons/hbase/templates/cmpd-hregionserver.yaml" 'podFQDNs: Optional'
259+
assert_contains "${ROOT_DIR}/addons/hadoop/templates/cmpd-hdfs-namenode.yaml" "fieldPath: metadata.annotations['kubeblocks.io/pod-fqdn']"
260+
assert_contains "${ROOT_DIR}/addons/hadoop/templates/cmpd-hdfs-namenode-standalone.yaml" "fieldPath: metadata.annotations['kubeblocks.io/pod-fqdn']"
245261
assert_contains "${ROOT_DIR}/addons/hbase/config/hbase-env.sh.tpl" '-Dhbase.regionserver.hostname=${POD_FQDN:-$(hostname -f 2>/dev/null || hostname)}'
246262
assert_contains "${ROOT_DIR}/addons/hbase/templates/cmpd-hmaster.yaml" 'optional: true'
247263
assert_contains "${ROOT_DIR}/addons/hbase/templates/cmpd-hmaster.yaml" 'podFQDNs: Optional'

0 commit comments

Comments
 (0)