feat: improve web installer node checks and disk configuration (#3162)

* feat(web): expose host blockdevices for web installer storage configuration

Signed-off-by: yongchuanzhou <18600490162@163.com>

* feat: add inventory host storage module

Support configuring host storage items from inventory for disk formatting, LVM volume management, and optional mount settings.

Signed-off-by: yongchuanzhou <18600490162@163.com>

* feat: add inventory host storage module

Support configuring host storage items from inventory for disk formatting, LVM volume management, and optional mount settings.

Signed-off-by: yongchuanzhou <18600490162@163.com>

* feat: add multipath storage support

Signed-off-by: yongchuanzhou <18600490162@163.com>

* fix: render storage module templates

Signed-off-by: yongchuanzhou <18600490162@163.com>

* fix: force cleanup selected storage disks

Signed-off-by: yongchuanzhou <18600490162@163.com>

* feat: expose gpu info in host check

Signed-off-by: yongchuanzhou <18600490162@163.com>

* fix: align web installer paths and host groups

Signed-off-by: yongchuanzhou <18600490162@163.com>

* fix: stabilize web host checks

Signed-off-by: yongchuanzhou <18600490162@163.com>

* fix: harden storage script quoting

Signed-off-by: yongchuanzhou <18600490162@163.com>

* Refine web installer host setup

Signed-off-by: yongchuanzhou <18600490162@163.com>

* Refine host check handling

Signed-off-by: yongchuanzhou <18600490162@163.com>

* refactor: move storage and multipath logic to roles per review

Signed-off-by: redscholar <blacktiledhouse@gmail.com>

* feat: support harbor port configuration

Signed-off-by: yongchuanzhou <18600490162@163.com>

* fix: preserve allocated storage devices

Signed-off-by: zyc <zyc@macdeMacBook-Pro-2.local>

* fix: load gpu vendor config from scanner path

Signed-off-by: yongchuanzhou <18600490162@163.com>

* fix: change harbor schema

Signed-off-by: redscholar <blacktiledhouse@gmail.com>

* fix: normalize harbor registry host

Signed-off-by: yongchuanzhou <18600490162@163.com>

* fix: use image_registry.http_port and image_registry.https_port

Signed-off-by: redscholar <blacktiledhouse@gmail.com>

* fix: add retries in when push images

Signed-off-by: redscholar <blacktiledhouse@gmail.com>

* fix: fix: handle empty http_code and merge harbor registry creation tasks

Signed-off-by: redscholar <blacktiledhouse@gmail.com>

* fix: use kubernetes.storage_disks instance of storage

Signed-off-by: redscholar <blacktiledhouse@gmail.com>

---------

Signed-off-by: yongchuanzhou <18600490162@163.com>
Signed-off-by: redscholar <blacktiledhouse@gmail.com>
Signed-off-by: zyc <zyc@macdeMacBook-Pro-2.local>
Co-authored-by: redscholar <blacktiledhouse@gmail.com>
Co-authored-by: zyc <zyc@macdeMacBook-Pro-2.local>

Author: 3 people

Makefile

@@ -190,7 +190,9 @@ generate-goimports:  ## Format all import, `goimports` is required.
 .PHONY: lint
 lint: $(GOLANGCI_LINT) ## Lint the codebase
 	@$(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS)
-	@cd $(TEST_DIR); $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS)
+	@if find $(TEST_DIR) -name '*.go' | grep -q .; then \
+		cd $(TEST_DIR); $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS); \
+	fi
 
 .PHONY: verify-dockerfiles
 verify-dockerfiles:

builtin/core/defaults/config/v1.23.yaml

@@ -33,14 +33,24 @@ spec:
     # Leave empty to skip installation (assuming an existing registry is available).
     type: ""
     # auth:
-    #   # Address of the private image registry
+    #   # Address of the private image registry. Format: host:port/project
+    #   # The port (if present) is used as http_port when plain_http=true,
+    #   # or as https_port when plain_http=false.
     #   registry: "dockerhub.kubekey.local"
+    #   # Use plain HTTP (no TLS). Default is false.
+    #   plain_http: false
+    #   username: admin
+    #   password: Harbor12345
     # Specify a VIP for image registry high availability. If set, enables HA.
     ha_vip: ""
     # ========== Image Registry - High Availability ==========
     # keepalived image tag for load balancing when multiple registry nodes exist.
     # keepalived_version: 2.3.4
     # ========== Image Registry - Harbor ==========
+    # Image registry port settings. Usually you do not need to set them manually;
+    # they are derived from image_registry.auth.registry and plain_http.
+    # http_port: 80
+    # https_port: 443
     # Harbor image tag (only valid if type is harbor).
     #harbor_version: v2.10.2
     # docker-compose binary

builtin/core/defaults/config/v1.24.yaml

@@ -33,14 +33,24 @@ spec:
     # Leave empty to skip installation (assuming an existing registry is available).
     type: ""
     # auth:
-    #   # Address of the private image registry
+    #   # Address of the private image registry. Format: host:port/project
+    #   # The port (if present) is used as http_port when plain_http=true,
+    #   # or as https_port when plain_http=false.
     #   registry: "dockerhub.kubekey.local"
+    #   # Use plain HTTP (no TLS). Default is false.
+    #   plain_http: false
+    #   username: admin
+    #   password: Harbor12345
     # Specify a VIP for image registry high availability. If set, enables HA.
     ha_vip: ""
     # ========== Image Registry - High Availability ==========
     # keepalived image tag for load balancing when multiple registry nodes exist.
     # keepalived_version: 2.3.4
     # ========== Image Registry - Harbor ==========
+    # Image registry port settings. Usually you do not need to set them manually;
+    # they are derived from image_registry.auth.registry and plain_http.
+    # http_port: 80
+    # https_port: 443
     # Harbor image tag (only valid if type is harbor).
     #harbor_version: v2.10.2
     # docker-compose binary

builtin/core/defaults/config/v1.25.yaml

@@ -33,14 +33,24 @@ spec:
     # Leave empty to skip installation (assuming an existing registry is available).
     type: ""
     # auth:
-    #   # Address of the private image registry
+    #   # Address of the private image registry. Format: host:port/project
+    #   # The port (if present) is used as http_port when plain_http=true,
+    #   # or as https_port when plain_http=false.
     #   registry: "dockerhub.kubekey.local"
+    #   # Use plain HTTP (no TLS). Default is false.
+    #   plain_http: false
+    #   username: admin
+    #   password: Harbor12345
     # Specify a VIP for image registry high availability. If set, enables HA.
     ha_vip: ""
     # ========== Image Registry - High Availability ==========
     # keepalived image tag for load balancing when multiple registry nodes exist.
     # keepalived_version: 2.3.4
     # ========== Image Registry - Harbor ==========
+    # Image registry port settings. Usually you do not need to set them manually;
+    # they are derived from image_registry.auth.registry and plain_http.
+    # http_port: 80
+    # https_port: 443
     # Harbor image tag (only valid if type is harbor).
     #harbor_version: v2.10.2
     # docker-compose binary

builtin/core/defaults/config/v1.26.yaml

@@ -33,14 +33,24 @@ spec:
     # Leave empty to skip installation (assuming an existing registry is available).
     type: ""
     # auth:
-    #   # Address of the private image registry
+    #   # Address of the private image registry. Format: host:port/project
+    #   # The port (if present) is used as http_port when plain_http=true,
+    #   # or as https_port when plain_http=false.
     #   registry: "dockerhub.kubekey.local"
+    #   # Use plain HTTP (no TLS). Default is false.
+    #   plain_http: false
+    #   username: admin
+    #   password: Harbor12345
     # Specify a VIP for image registry high availability. If set, enables HA.
     ha_vip: ""
     # ========== Image Registry - High Availability ==========
     # keepalived image tag for load balancing when multiple registry nodes exist.
     # keepalived_version: 2.3.4
     # ========== Image Registry - Harbor ==========
+    # Image registry port settings. Usually you do not need to set them manually;
+    # they are derived from image_registry.auth.registry and plain_http.
+    # http_port: 80
+    # https_port: 443
     # Harbor image tag (only valid if type is harbor).
     #harbor_version: v2.10.2
     # docker-compose binary

builtin/core/defaults/config/v1.27.yaml

@@ -33,14 +33,24 @@ spec:
     # Leave empty to skip installation (assuming an existing registry is available).
     type: ""
     # auth:
-    #   # Address of the private image registry
+    #   # Address of the private image registry. Format: host:port/project
+    #   # The port (if present) is used as http_port when plain_http=true,
+    #   # or as https_port when plain_http=false.
     #   registry: "dockerhub.kubekey.local"
+    #   # Use plain HTTP (no TLS). Default is false.
+    #   plain_http: false
+    #   username: admin
+    #   password: Harbor12345
     # Specify a VIP for image registry high availability. If set, enables HA.
     ha_vip: ""
     # ========== Image Registry - High Availability ==========
     # keepalived image tag for load balancing when multiple registry nodes exist.
     # keepalived_version: 2.3.4
     # ========== Image Registry - Harbor ==========
+    # Image registry port settings. Usually you do not need to set them manually;
+    # they are derived from image_registry.auth.registry and plain_http.
+    # http_port: 80
+    # https_port: 443
     # Harbor image tag (only valid if type is harbor).
     #harbor_version: v2.10.2
     # docker-compose binary

builtin/core/defaults/config/v1.28.yaml

@@ -33,14 +33,24 @@ spec:
     # Leave empty to skip installation (assuming an existing registry is available).
     type: ""
     # auth:
-    #   # Address of the private image registry
+    #   # Address of the private image registry. Format: host:port/project
+    #   # The port (if present) is used as http_port when plain_http=true,
+    #   # or as https_port when plain_http=false.
     #   registry: "dockerhub.kubekey.local"
+    #   # Use plain HTTP (no TLS). Default is false.
+    #   plain_http: false
+    #   username: admin
+    #   password: Harbor12345
     # Specify a VIP for image registry high availability. If set, enables HA.
     ha_vip: ""
     # ========== Image Registry - High Availability ==========
     # keepalived image tag for load balancing when multiple registry nodes exist.
     # keepalived_version: 2.3.4
     # ========== Image Registry - Harbor ==========
+    # Image registry port settings. Usually you do not need to set them manually;
+    # they are derived from image_registry.auth.registry and plain_http.
+    # http_port: 80
+    # https_port: 443
     # Harbor image tag (only valid if type is harbor).
     #harbor_version: v2.10.2
     # docker-compose binary

builtin/core/defaults/config/v1.29.yaml

@@ -33,14 +33,24 @@ spec:
     # Leave empty to skip installation (assuming an existing registry is available).
     type: ""
     # auth:
-    #   # Address of the private image registry
+    #   # Address of the private image registry. Format: host:port/project
+    #   # The port (if present) is used as http_port when plain_http=true,
+    #   # or as https_port when plain_http=false.
     #   registry: "dockerhub.kubekey.local"
+    #   # Use plain HTTP (no TLS). Default is false.
+    #   plain_http: false
+    #   username: admin
+    #   password: Harbor12345
     # Specify a VIP for image registry high availability. If set, enables HA.
     ha_vip: ""
     # ========== Image Registry - High Availability ==========
     # keepalived image tag for load balancing when multiple registry nodes exist.
     # keepalived_version: 2.3.4
     # ========== Image Registry - Harbor ==========
+    # Image registry port settings. Usually you do not need to set them manually;
+    # they are derived from image_registry.auth.registry and plain_http.
+    # http_port: 80
+    # https_port: 443
     # Harbor image tag (only valid if type is harbor).
     #harbor_version: v2.10.2
     # docker-compose binary

builtin/core/defaults/config/v1.30.yaml

@@ -33,14 +33,24 @@ spec:
     # Leave empty to skip installation (assuming an existing registry is available).
     type: ""
     # auth:
-    #   # Address of the private image registry
+    #   # Address of the private image registry. Format: host:port/project
+    #   # The port (if present) is used as http_port when plain_http=true,
+    #   # or as https_port when plain_http=false.
     #   registry: "dockerhub.kubekey.local"
+    #   # Use plain HTTP (no TLS). Default is false.
+    #   plain_http: false
+    #   username: admin
+    #   password: Harbor12345
     # Specify a VIP for image registry high availability. If set, enables HA.
     ha_vip: ""
     # ========== Image Registry - High Availability ==========
     # keepalived image tag for load balancing when multiple registry nodes exist.
     # keepalived_version: 2.3.4
     # ========== Image Registry - Harbor ==========
+    # Image registry port settings. Usually you do not need to set them manually;
+    # they are derived from image_registry.auth.registry and plain_http.
+    # http_port: 80
+    # https_port: 443
     # Harbor image tag (only valid if type is harbor).
     #harbor_version: v2.10.2
     # docker-compose binary

builtin/core/defaults/config/v1.31.yaml

@@ -33,14 +33,24 @@ spec:
     # Leave empty to skip installation (assuming an existing registry is available).
     type: ""
     # auth:
-    #   # Address of the private image registry
+    #   # Address of the private image registry. Format: host:port/project
+    #   # The port (if present) is used as http_port when plain_http=true,
+    #   # or as https_port when plain_http=false.
     #   registry: "dockerhub.kubekey.local"
+    #   # Use plain HTTP (no TLS). Default is false.
+    #   plain_http: false
+    #   username: admin
+    #   password: Harbor12345
     # Specify a VIP for image registry high availability. If set, enables HA.
     ha_vip: ""
     # ========== Image Registry - High Availability ==========
     # keepalived image tag for load balancing when multiple registry nodes exist.
     # keepalived_version: 2.3.4
     # ========== Image Registry - Harbor ==========
+    # Image registry port settings. Usually you do not need to set them manually;
+    # they are derived from image_registry.auth.registry and plain_http.
+    # http_port: 80
+    # https_port: 443
     # Harbor image tag (only valid if type is harbor).
     #harbor_version: v2.10.2
     # docker-compose binary